diff --git a/.circleci/config.yml b/.circleci/config.yml index c614d3f0e..23466cf48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,8 @@ commands: - run: name: Install JDK 8 on macos command: | - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8 + HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8 increase-max-open-files-on-macos: steps: @@ -44,6 +45,7 @@ commands: echo "export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $BASH_ENV echo "export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $BASH_ENV echo "export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $BASH_ENV + echo "export DISABLE_PERF_CONTEXT=0" >> $BASH_ENV windows-build-steps: steps: @@ -53,6 +55,7 @@ commands: command: | echo "Installing CMake..." choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y + choco install liberica8jdk -y mkdir $Env:THIRDPARTY_HOME cd $Env:THIRDPARTY_HOME echo "Building Snappy dependency..." @@ -66,9 +69,10 @@ commands: - run: name: "Build RocksDB" command: | + $env:Path = $env:JAVA_HOME + ";" + $env:Path mkdir build cd build - & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 .. + & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 .. cd .. echo "Building with VS version: $Env:CMAKE_GENERATOR" msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 @@ -77,6 +81,11 @@ commands: shell: powershell.exe command: | build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16 + - run: + name: "Test RocksJava" + command: | + cd build\java + & $Env:CTEST_BIN -C Debug -j 16 pre-steps-macos: steps: - pre-steps @@ -104,6 +113,15 @@ commands: path: /tmp/core_dumps when: on_fail + post-pmd-steps: + steps: + - store_artifacts: + path: /home/circleci/project/java/target/pmd.xml + when: on_fail + - store_artifacts: + path: /home/circleci/project/java/target/site + when: on_fail + upgrade-cmake: steps: - run: @@ -112,6 +130,17 @@ commands: sudo apt remove --purge cmake sudo snap install cmake --classic + install-clang-10: + steps: + - run: + name: Install Clang 10 + command: | + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list + echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list + echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable + sudo apt-get update -y && sudo apt-get install -y clang-10 + install-gflags: steps: - run: @@ -126,6 +155,13 @@ commands: command: | HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags + install-maven: + steps: + - run: + name: Install maven + command: | + sudo apt-get update -y && sudo apt-get install -y maven + setup-folly: steps: - run: @@ -205,25 +241,30 @@ executors: # $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash # option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process - image: zjay437/rocksdb:0.6 + linux-java-docker: + docker: + - image: evolvedbinary/rocksjava:centos6_x64-be jobs: build-macos: macos: - xcode: 12.5.1 - resource_class: large + xcode: 14.3.1 + resource_class: macos.m1.medium.gen1 environment: ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now steps: - increase-max-open-files-on-macos - install-gflags-on-macos - pre-steps-macos - - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=32 -j32 all + - run: + command: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=16 -j16 all + no_output_timeout: 30m - post-steps build-macos-cmake: macos: - xcode: 12.5.1 - resource_class: large + xcode: 14.3.1 + resource_class: macos.m1.medium.gen1 parameters: run_even_tests: description: run even or odd tests, used to split tests to 2 groups @@ -239,67 +280,67 @@ jobs: command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. - run: name: "Build tests" - command: cd build && make V=1 -j32 + command: cd build && make V=1 -j16 - when: condition: << parameters.run_even_tests >> steps: - run: name: "Run even tests" - command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 0,,2 + command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2 - when: condition: not: << parameters.run_even_tests >> steps: - run: name: "Run odd tests" - command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 1,,2 + command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2 - post-steps build-linux: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: make V=1 J=32 -j32 check + - run: make V=1 J=8 -j8 check - post-steps build-linux-encrypted_env-no_compression: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check + - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=8 -j8 check - run: | ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression - post-steps build-linux-static_lib-alt_namespace-status_checked: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j8 check - post-steps build-linux-release: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - checkout # check out the code in the project directory - - run: make V=1 -j32 LIB_MODE=shared release + - run: make V=1 -j8 LIB_MODE=shared release - run: ls librocksdb.so # ensure shared lib built - run: ./db_stress --version # ensure with gflags - run: make clean - - run: make V=1 -j32 release + - run: make V=1 -j8 release - run: ls librocksdb.a # ensure static lib built - run: ./db_stress --version # ensure with gflags - run: make clean - run: apt-get remove -y libgflags-dev - - run: make V=1 -j32 LIB_MODE=shared release + - run: make V=1 -j8 LIB_MODE=shared release - run: ls librocksdb.so # ensure shared lib built - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - run: make clean - - run: make V=1 -j32 release + - run: make V=1 -j8 release - run: ls librocksdb.a # ensure static lib built - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - post-steps @@ -321,47 +362,53 @@ jobs: resource_class: xlarge steps: - checkout # check out the code in the project directory - - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all + - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 DISABLE_PERF_CONTEXT=0 make V=1 -j16 all - post-steps build-linux-clang10-asan: - executor: linux-docker - resource_class: 2xlarge + machine: + image: ubuntu-2004:202111-02 + resource_class: xlarge steps: - pre-steps + - install-gflags + - install-clang-10 - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out - post-steps build-linux-clang10-mini-tsan: - executor: linux-docker - resource_class: 2xlarge+ + machine: + image: ubuntu-2004:202111-02 + resource_class: xlarge steps: - pre-steps - - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - install-gflags + - install-clang-10 + - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out. - post-steps build-linux-clang10-ubsan: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out + - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j8 ubsan_check # aligned new doesn't work for reason we haven't figured out - post-steps build-linux-valgrind: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: PORTABLE=1 make V=1 -j32 valgrind_test + - run: PORTABLE=1 make V=1 -j8 valgrind_test - post-steps build-linux-clang10-clang-analyze: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j8 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. - post-steps - run: name: "compress test report" @@ -386,29 +433,35 @@ jobs: build-linux-cmake-with-folly: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge + environment: + CC: gcc-10 + CXX: g++-10 steps: - pre-steps - setup-folly - build-folly - - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20) + - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j8 && ctest -j8) - post-steps build-linux-cmake-with-folly-lite-no-test: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge + environment: + CC: gcc-10 + CXX: g++-10 steps: - pre-steps - setup-folly - - run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20) + - run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j8) - post-steps build-linux-cmake-with-benchmark: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 + - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j8 && ctest -j8 - post-steps build-linux-unity-and-headers: @@ -422,41 +475,47 @@ jobs: - run: apt-get update -y && apt-get install -y libgflags-dev - run: name: "Unity build" - command: make V=1 -j8 unity_test + command: DISABLE_PERF_CONTEXT=0 make V=1 -j8 unity_test no_output_timeout: 20m - run: make V=1 -j8 -k check-headers # could be moved to a different build - post-steps - build-linux-gcc-7-with-folly: + build-linux-make-with-folly: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge + environment: + CC: gcc-10 + CXX: g++-10 steps: - pre-steps - setup-folly - build-folly - - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures + - run: USE_FOLLY=1 LIB_MODE=static V=1 make -j8 check # TODO: LIB_MODE only to work around unresolved linker failures - post-steps - build-linux-gcc-7-with-folly-lite-no-test: + build-linux-make-with-folly-lite-no-test: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge + environment: + CC: gcc-10 + CXX: g++-10 steps: - pre-steps - setup-folly - - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - run: USE_FOLLY_LITE=1 V=1 make -j8 all - post-steps build-linux-gcc-8-no_test_run: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all + - run: CC=gcc-8 CXX=g++-8 V=1 make -j8 all - post-steps build-linux-cmake-with-folly-coroutines: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge environment: CC: gcc-10 CXX: g++-10 @@ -464,51 +523,51 @@ jobs: - pre-steps - setup-folly - build-folly - - run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20) + - run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j8 && ctest -j8) - post-steps build-linux-gcc-10-cxx20-no_test_run: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all + - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j8 all - post-steps build-linux-gcc-11-no_test_run: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench # TODO: LIB_MODE only to work around unresolved linker failures + - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j8 all microbench # TODO: LIB_MODE only to work around unresolved linker failures - post-steps build-linux-clang-13-no_test_run: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench + - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j8 all microbench - post-steps # Ensure ASAN+UBSAN with folly, and full testsuite with clang 13 build-linux-clang-13-asan-ubsan-with-folly: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - setup-folly - build-folly - - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures + - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j8 check # TODO: LIB_MODE only to work around unresolved linker failures - post-steps # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing. build-linux-run-microbench: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - - run: DEBUG_LEVEL=0 make -j32 run_microbench + - run: DEBUG_LEVEL=0 make -j8 run_microbench - post-steps build-linux-mini-crashtest: @@ -521,53 +580,77 @@ jobs: build-linux-crashtest-tiered-storage-bb: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - run: name: "run crashtest" - command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' blackbox_crash_test_with_tiered_storage + command: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' blackbox_crash_test_with_tiered_storage no_output_timeout: 100m - post-steps build-linux-crashtest-tiered-storage-wb: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - run: name: "run crashtest" - command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' whitebox_crash_test_with_tiered_storage + command: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' whitebox_crash_test_with_tiered_storage no_output_timeout: 100m - post-steps + build-windows-vs2022-avx2: + executor: + name: win/server-2022 + size: xlarge + environment: + THIRDPARTY_HOME: C:/Users/circleci/thirdparty + CMAKE_HOME: C:/Program Files/CMake + CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe + JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8 + SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8 + SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build + SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: AVX2 + steps: + - windows-build-steps + build-windows-vs2022: executor: name: win/server-2022 - size: 2xlarge + size: xlarge environment: THIRDPARTY_HOME: C:/Users/circleci/thirdparty CMAKE_HOME: C:/Program Files/CMake CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe + JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8 SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8 SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: 1 steps: - windows-build-steps build-windows-vs2019: executor: name: win/server-2019 - size: 2xlarge + size: xlarge environment: THIRDPARTY_HOME: C:/Users/circleci/thirdparty CMAKE_HOME: C:/Program Files/CMake CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe + JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8 SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8 SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib CMAKE_GENERATOR: Visual Studio 16 2019 + CMAKE_PORTABLE: 1 steps: - windows-build-steps @@ -588,8 +671,29 @@ jobs: command: make V=1 J=8 -j8 jtest - post-steps +# build-linux-java-pmd: +# machine: +# image: ubuntu-2004:202111-02 +# resource_class: large +# environment: +# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 +# steps: +# - install-maven +# - pre-steps +# - run: +# name: "Set Java Environment" +# command: | +# echo "JAVA_HOME=${JAVA_HOME}" +# echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV +# which java && java -version +# which javac && javac -version +# - run: +# name: "PMD RocksDBJava" +# command: make V=1 J=8 -j8 jpmd +# - post-pmd-steps + build-linux-java-static: - executor: linux-docker + executor: linux-java-docker resource_class: large steps: - pre-steps @@ -602,15 +706,15 @@ jobs: which javac && javac -version - run: name: "Build RocksDBJava Static Library" - command: make V=1 J=8 -j8 rocksdbjavastatic + command: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' - post-steps build-macos-java: macos: - xcode: 12.5.1 - resource_class: large + xcode: 14.3.1 + resource_class: macos.m1.medium.gen1 environment: - JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home + JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash steps: - increase-max-open-files-on-macos @@ -632,10 +736,10 @@ jobs: build-macos-java-static: macos: - xcode: 12.5.1 - resource_class: large + xcode: 14.3.1 + resource_class: macos.m1.medium.gen1 environment: - JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home + JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home steps: - increase-max-open-files-on-macos - install-gflags-on-macos @@ -657,10 +761,10 @@ jobs: build-macos-java-static-universal: macos: - xcode: 12.5.1 - resource_class: large + xcode: 14.3.1 + resource_class: macos.m1.medium.gen1 environment: - JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home + JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home steps: - increase-max-open-files-on-macos - install-gflags-on-macos @@ -709,12 +813,12 @@ jobs: build-linux-non-shm: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge environment: TEST_TMPDIR: /tmp/rocksdb_test_tmp steps: - pre-steps - - run: make V=1 -j32 check + - run: make V=1 -j8 check - post-steps build-linux-arm-test-full: @@ -772,7 +876,7 @@ jobs: build-format-compatible: executor: linux-docker - resource_class: 2xlarge + resource_class: xlarge steps: - pre-steps - run: @@ -812,8 +916,8 @@ workflows: - build-linux - build-linux-cmake-with-folly - build-linux-cmake-with-folly-lite-no-test - - build-linux-gcc-7-with-folly - - build-linux-gcc-7-with-folly-lite-no-test + - build-linux-make-with-folly + - build-linux-make-with-folly-lite-no-test - build-linux-cmake-with-folly-coroutines - build-linux-cmake-with-benchmark - build-linux-encrypted_env-no_compression @@ -842,7 +946,6 @@ workflows: - build-linux-mini-crashtest jobs-windows: jobs: - - build-windows-vs2022 - build-windows-vs2019 - build-cmake-mingw jobs-java: @@ -852,6 +955,7 @@ workflows: - build-macos-java - build-macos-java-static - build-macos-java-static-universal +# - build-linux-java-pmd jobs-macos: jobs: - build-macos @@ -888,5 +992,7 @@ workflows: - build-linux-arm-test-full - build-linux-run-microbench - build-linux-non-shm - - build-linux-clang-13-asan-ubsan-with-folly +# - build-linux-clang-13-asan-ubsan-with-folly - build-linux-valgrind + - build-windows-vs2022-avx2 + - build-windows-vs2022 diff --git a/.gitignore b/.gitignore index 8dd7e8296..8bd9fea59 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,7 @@ fbcode/ fbcode buckifier/*.pyc buckifier/__pycache__ +.arcconfig compile_commands.json clang-format-diff.py diff --git a/CMakeLists.txt b/CMakeLists.txt index fa630f327..9a0ed90c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -254,7 +254,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") set(PORTABLE 0 CACHE STRING "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU") -if(PORTABLE STREQUAL 1) +if(PORTABLE MATCHES "1|ON|YES|TRUE|Y") # Usually nothing to do; compiler default is typically the most general if(NOT MSVC) if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x") @@ -264,14 +264,7 @@ if(PORTABLE STREQUAL 1) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64") endif() endif() -elseif(PORTABLE MATCHES [^0]+) - # Name of a CPU arch spec or feature set to require - if(MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}") - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}") - endif() -else() +elseif(PORTABLE MATCHES "0|OFF|NO|FALSE|N") if(MSVC) # NOTE: No auto-detection of current CPU, but instead assume some useful # level of optimization is supported @@ -285,6 +278,13 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") endif() endif() +else() + # Name of a CPU arch spec or feature set to require + if(MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}") + endif() endif() include(CheckCXXSourceCompiles) @@ -596,7 +596,7 @@ if(USE_FOLLY) FMT_INST_PATH) exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE GFLAGS_INST_PATH) - set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.78.0) + set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.83.0) if(EXISTS ${FMT_INST_PATH}/lib64) set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt) else() @@ -632,6 +632,7 @@ set(SOURCES cache/secondary_cache.cc cache/secondary_cache_adapter.cc cache/sharded_cache.cc + cache/tiered_secondary_cache.cc db/arena_wrapped_db_iter.cc db/blob/blob_contents.cc db/blob/blob_fetcher.cc @@ -714,6 +715,7 @@ set(SOURCES db/wal_manager.cc db/wide/wide_column_serialization.cc db/wide/wide_columns.cc + db/wide/wide_columns_helper.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc @@ -725,6 +727,10 @@ set(SOURCES env/env_encryption.cc env/file_system.cc env/file_system_tracer.cc + env/flink/env_flink.cc + env/flink/jvm_util.cc + env/flink/jni_helper.cc + env/flink/env_flink_test_suite.cc env/fs_remap.cc env/mock_env.cc env/unique_id_gen.cc @@ -774,6 +780,7 @@ set(SOURCES options/configurable.cc options/customizable.cc options/db_options.cc + options/offpeak_time_info.cc options/options.cc options/options_helper.cc options/options_parser.cc @@ -1027,6 +1034,7 @@ if(USE_FOLLY_LITE) list(APPEND SOURCES third-party/folly/folly/container/detail/F14Table.cpp third-party/folly/folly/detail/Futex.cpp + third-party/folly/folly/lang/Exception.cpp third-party/folly/folly/lang/SafeAssert.cpp third-party/folly/folly/lang/ToAscii.cpp third-party/folly/folly/ScopeGuard.cpp @@ -1034,6 +1042,12 @@ if(USE_FOLLY_LITE) third-party/folly/folly/synchronization/DistributedMutex.cpp third-party/folly/folly/synchronization/ParkingLot.cpp) include_directories(${PROJECT_SOURCE_DIR}/third-party/folly) + exec_program(python3 ${PROJECT_SOURCE_DIR}/third-party/folly ARGS + build/fbcode_builder/getdeps.py show-source-dir boost OUTPUT_VARIABLE + BOOST_SOURCE_PATH) + exec_program(ls ARGS -d ${BOOST_SOURCE_PATH}/boost* OUTPUT_VARIABLE + BOOST_INCLUDE_DIR) + include_directories(${BOOST_INCLUDE_DIR}) add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG) list(APPEND THIRDPARTY_LIBS glog) endif() @@ -1108,11 +1122,15 @@ set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc) configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY) add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES} ${BUILD_VERSION_CC}) +target_include_directories(${ROCKSDB_STATIC_LIB} PUBLIC + $) target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) if(ROCKSDB_BUILD_SHARED) add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES} ${BUILD_VERSION_CC}) + target_include_directories(${ROCKSDB_SHARED_LIB} PUBLIC + $) target_link_libraries(${ROCKSDB_SHARED_LIB} PRIVATE ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) @@ -1272,6 +1290,7 @@ if(WITH_TESTS) cache/cache_test.cc cache/compressed_secondary_cache_test.cc cache/lru_cache_test.cc + cache/tiered_secondary_cache_test.cc db/blob/blob_counting_iterator_test.cc db/blob/blob_file_addition_test.cc db/blob/blob_file_builder_test.cc @@ -1367,6 +1386,7 @@ if(WITH_TESTS) db/wal_edit_test.cc db/wide/db_wide_basic_test.cc db/wide/wide_column_serialization_test.cc + db/wide/wide_columns_helper_test.cc db/write_batch_test.cc db/write_callback_test.cc db/write_controller_test.cc @@ -1442,6 +1462,7 @@ if(WITH_TESTS) utilities/cassandra/cassandra_format_test.cc utilities/cassandra/cassandra_row_merge_test.cc utilities/cassandra/cassandra_serialize_test.cc + utilities/flink/flink_compaction_filter_test.cc utilities/checkpoint/checkpoint_test.cc utilities/env_timed_test.cc utilities/flink/flink_compaction_filter_test.cc @@ -1603,6 +1624,3 @@ option(WITH_BENCHMARK "build benchmark tests" OFF) if(WITH_BENCHMARK) add_subdirectory(${PROJECT_SOURCE_DIR}/microbench/) endif() - -target_include_directories(${PROJECT_NAME} PUBLIC - $) diff --git a/HISTORY.md b/HISTORY.md new file mode 100644 index 000000000..e69de29bb diff --git a/INSTALL.md b/INSTALL.md index f4bb7e62a..fb4651e4b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -17,7 +17,7 @@ There are few options when compiling RocksDB: * `make check` will compile and run all the unit tests. `make check` will compile RocksDB in debug mode. * `make all` will compile our static library, and all our tools and unit tests. Our tools -depend on gflags. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't +depend on gflags 2.2.0 or newer. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't use binaries compiled by `make all` in production. * By default the binary we produce is optimized for the CPU you're compiling on @@ -77,7 +77,7 @@ most processors made since roughly 2013. git clone https://github.com/gflags/gflags.git cd gflags - git checkout v2.0 + git checkout v2.2.0 ./configure && make && sudo make install **Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the diff --git a/Makefile b/Makefile index 644a4b4df..fa6948417 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ #----------------------------------------------- -FORST_VERSION ?= 0.1.0 +FORST_VERSION ?= 0.1.2-beta BASH_EXISTS := $(shell which bash) SHELL := $(shell which bash) @@ -96,6 +96,26 @@ endif $(info $$DEBUG_LEVEL is $(DEBUG_LEVEL), $$LIB_MODE is $(LIB_MODE)) +# Detect what platform we're building on. +# Export some common variables that might have been passed as Make variables +# instead of environment variables. +dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ + export CXXFLAGS="$(EXTRA_CXXFLAGS)"; \ + export LDFLAGS="$(EXTRA_LDFLAGS)"; \ + export COMPILE_WITH_ASAN="$(COMPILE_WITH_ASAN)"; \ + export COMPILE_WITH_TSAN="$(COMPILE_WITH_TSAN)"; \ + export COMPILE_WITH_UBSAN="$(COMPILE_WITH_UBSAN)"; \ + export PORTABLE="$(PORTABLE)"; \ + export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \ + export USE_CLANG="$(USE_CLANG)"; \ + export LIB_MODE="$(LIB_MODE)"; \ + export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \ + export USE_FOLLY="$(USE_FOLLY)"; \ + export USE_FOLLY_LITE="$(USE_FOLLY_LITE)"; \ + "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) +# this file is generated by the previous line to set build flags and sources +include make_config.mk + # Figure out optimize level. ifneq ($(DEBUG_LEVEL), 2) OPTIMIZE_LEVEL ?= -O2 @@ -231,25 +251,6 @@ am__v_AR_1 = AM_LINK = $(AM_V_CCLD)$(CXX) -L. $(patsubst lib%.a, -l%, $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^)) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) AM_SHARE = $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$@ -L. $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^) $(EXEC_LDFLAGS) $(LDFLAGS) -o $@ -# Detect what platform we're building on. -# Export some common variables that might have been passed as Make variables -# instead of environment variables. -dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ - export CXXFLAGS="$(EXTRA_CXXFLAGS)"; \ - export LDFLAGS="$(EXTRA_LDFLAGS)"; \ - export COMPILE_WITH_ASAN="$(COMPILE_WITH_ASAN)"; \ - export COMPILE_WITH_TSAN="$(COMPILE_WITH_TSAN)"; \ - export COMPILE_WITH_UBSAN="$(COMPILE_WITH_UBSAN)"; \ - export PORTABLE="$(PORTABLE)"; \ - export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \ - export USE_CLANG="$(USE_CLANG)"; \ - export LIB_MODE="$(LIB_MODE)"; \ - export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \ - export USE_FOLLY="$(USE_FOLLY)"; \ - "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) -# this file is generated by the previous line to set build flags and sources -include make_config.mk - ROCKSDB_PLUGIN_MKS = $(foreach plugin, $(ROCKSDB_PLUGINS), plugin/$(plugin)/*.mk) include $(ROCKSDB_PLUGIN_MKS) ROCKSDB_PLUGIN_PROTO =ROCKSDB_NAMESPACE::ObjectLibrary\&, const std::string\& @@ -434,6 +435,12 @@ ifndef DISABLE_JEMALLOC PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE) endif +DISABLE_PERF_CONTEXT ?= 1 +ifeq ($(DISABLE_PERF_CONTEXT),1) + PLATFORM_CXXFLAGS += -DNPERF_CONTEXT + PLATFORM_CCFLAGS += -DNPERF_CONTEXT +endif + ifndef USE_FOLLY USE_FOLLY=0 endif @@ -508,6 +515,17 @@ endif ifeq ($(USE_FOLLY_LITE),1) # Path to the Folly source code and include files FOLLY_DIR = ./third-party/folly +ifneq ($(strip $(BOOST_SOURCE_PATH)),) + BOOST_INCLUDE = $(shell (ls -d $(BOOST_SOURCE_PATH)/boost*/)) + # AIX: pre-defined system headers are surrounded by an extern "C" block + ifeq ($(PLATFORM), OS_AIX) + PLATFORM_CCFLAGS += -I$(BOOST_INCLUDE) + PLATFORM_CXXFLAGS += -I$(BOOST_INCLUDE) + else + PLATFORM_CCFLAGS += -isystem $(BOOST_INCLUDE) + PLATFORM_CXXFLAGS += -isystem $(BOOST_INCLUDE) + endif +endif # BOOST_SOURCE_PATH # AIX: pre-defined system headers are surrounded by an extern "C" block ifeq ($(PLATFORM), OS_AIX) PLATFORM_CCFLAGS += -I$(FOLLY_DIR) @@ -547,7 +565,7 @@ endif ifdef USE_CLANG # Used by some teams in Facebook - WARNING_FLAGS += -Wshift-sign-overflow -Wambiguous-reversed-operator + WARNING_FLAGS += -Wshift-sign-overflow -Wambiguous-reversed-operator -Wimplicit-fallthrough endif ifeq ($(PLATFORM), OS_OPENBSD) @@ -1002,7 +1020,7 @@ endif .PHONY: check_0 check_0: - printf '%s\n' '' \ + @printf '%s\n' '' \ 'To monitor subtest ,' \ ' run "make watch-log" in a separate window' ''; \ { \ @@ -1024,7 +1042,7 @@ valgrind-exclude-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest. .PHONY: valgrind_check_0 valgrind_check_0: test_log_prefix := valgrind_ valgrind_check_0: - printf '%s\n' '' \ + @printf '%s\n' '' \ 'To monitor subtest ,' \ ' run "make watch-log" in a separate window' ''; \ { \ @@ -1896,6 +1914,9 @@ compressed_secondary_cache_test: $(OBJ_DIR)/cache/compressed_secondary_cache_tes lru_cache_test: $(OBJ_DIR)/cache/lru_cache_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) +tiered_secondary_cache_test: $(OBJ_DIR)/cache/tiered_secondary_cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + range_del_aggregator_test: $(OBJ_DIR)/db/range_del_aggregator_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) @@ -1995,6 +2016,9 @@ cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test. wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) +wide_columns_helper_test: $(OBJ_DIR)/db/wide/wide_columns_helper_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + #------------------------------------------------- # make install related stuff PREFIX ?= /usr/local @@ -2065,7 +2089,7 @@ JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux ifeq ($(PLATFORM), OS_SOLARIS) ARCH := $(shell isainfo -b) else ifeq ($(PLATFORM), OS_OPENBSD) - ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 sparc64 loongarch64, $(MACHINE))) + ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 riscv64 sparc64 loongarch64, $(MACHINE))) ARCH := 64 else ARCH := 32 @@ -2085,48 +2109,49 @@ ifneq ($(origin JNI_LIBC), undefined) JNI_LIBC_POSTFIX = -$(JNI_LIBC) endif -ifeq (,$(ROCKSDBJNILIB)) -ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64 loongarch64, $(MACHINE))) - ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so +ifeq (,$(FORSTJNILIB)) +ifneq (,$(filter ppc% s390x arm64 aarch64 riscv64 sparc64 loongarch64, $(MACHINE))) + FORSTJNILIB = libforstjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so else - ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so + FORSTJNILIB = libforstjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so endif endif ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH) -ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux$(ARCH)$(JNI_LIBC_POSTFIX).jar -ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar -ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-sources.jar +ROCKSDB_JAVA_VERSION = $(FORST_VERSION) +ROCKSDB_JAR = forstjni-$(ROCKSDB_JAVA_VERSION)-linux$(ARCH)$(JNI_LIBC_POSTFIX).jar +ROCKSDB_JAR_ALL = forstjni-$(ROCKSDB_JAVA_VERSION).jar +ROCKSDB_JAVADOCS_JAR = forstjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar +ROCKSDB_SOURCES_JAR = forstjni-$(ROCKSDB_JAVA_VERSION)-sources.jar SHA256_CMD = sha256sum -ZLIB_VER ?= 1.3.1 -ZLIB_SHA256 ?= 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 -ZLIB_DOWNLOAD_BASE ?= http://zlib.net +ZLIB_VER ?= 1.3 +ZLIB_SHA256 ?= ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e +ZLIB_DOWNLOAD_BASE ?= https://zlib.net/fossils BZIP2_VER ?= 1.0.8 BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 BZIP2_DOWNLOAD_BASE ?= http://sourceware.org/pub/bzip2 SNAPPY_VER ?= 1.1.8 SNAPPY_SHA256 ?= 16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive -LZ4_VER ?= 1.9.3 -LZ4_SHA256 ?= 030644df4611007ff7dc962d981f390361e6c97a34e5cbc393ddfbe019ffe2c1 +LZ4_VER ?= 1.9.4 +LZ4_SHA256 ?= 0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive -ZSTD_VER ?= 1.4.9 -ZSTD_SHA256 ?= acf714d98e3db7b876e5b540cbf6dee298f60eb3c0723104f6d3f065cd60d6a8 +ZSTD_VER ?= 1.5.5 +ZSTD_SHA256 ?= 98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1 ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive CURL_SSL_OPTS ?= --tlsv1 ifeq ($(PLATFORM), OS_MACOSX) -ifeq (,$(findstring librocksdbjni-osx,$(ROCKSDBJNILIB))) +ifeq (,$(findstring libforstjni-osx,$(FORSTJNILIB))) ifeq ($(MACHINE),arm64) - ROCKSDBJNILIB = librocksdbjni-osx-arm64.jnilib + FORSTJNILIB = libforstjni-osx-arm64.jnilib else ifeq ($(MACHINE),x86_64) - ROCKSDBJNILIB = librocksdbjni-osx-x86_64.jnilib + FORSTJNILIB = libforstjni-osx-x86_64.jnilib else - ROCKSDBJNILIB = librocksdbjni-osx.jnilib + FORSTJNILIB = libforstjni-osx.jnilib endif endif - ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-osx.jar + ROCKSDB_JAR = forstjni-$(ROCKSDB_JAVA_VERSION)-osx.jar SHA256_CMD = openssl sha256 -r ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","") JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin @@ -2137,25 +2162,25 @@ endif ifeq ($(PLATFORM), OS_FREEBSD) JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/freebsd - ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so - ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-freebsd$(ARCH).jar + FORSTJNILIB = libforstjni-freebsd$(ARCH).so + ROCKSDB_JAR = forstjni-$(ROCKSDB_JAVA_VERSION)-freebsd$(ARCH).jar endif ifeq ($(PLATFORM), OS_SOLARIS) - ROCKSDBJNILIB = librocksdbjni-solaris$(ARCH).so - ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar + FORSTJNILIB = libforstjni-solaris$(ARCH).so + ROCKSDB_JAR = forstjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris SHA256_CMD = digest -a sha256 endif ifeq ($(PLATFORM), OS_AIX) JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/aix - ROCKSDBJNILIB = librocksdbjni-aix.so + FORSTJNILIB = libforstjni-aix.so EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf - SNAPPY_MAKE_TARGET = libsnappy.la endif ifeq ($(PLATFORM), OS_OPENBSD) JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/openbsd - ROCKSDBJNILIB = librocksdbjni-openbsd$(ARCH).so - ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-openbsd$(ARCH).jar + FORSTJNILIB = libforstjni-openbsd$(ARCH).so + ROCKSDB_JAR = forstjni-$(ROCKSDB_JAVA_VERSION)-openbsd$(ARCH).jar endif export SHA256_CMD @@ -2257,15 +2282,15 @@ endif rocksdbjavastaticosx: rocksdbjavastaticosx_archs cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md - cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib - cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) libforstjni-osx-x86_64.jnilib libforstjni-osx-arm64.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/forstdb/*.class org/forstdb/util/*.class openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs - cd java/target; lipo -create -output librocksdbjni-osx.jnilib librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib + cd java/target; lipo -create -output libforstjni-osx.jnilib libforstjni-osx-x86_64.jnilib libforstjni-osx-arm64.jnilib cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md - cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx.jnilib - cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) libforstjni-osx.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/forstdb/*.class org/forstdb/util/*.class openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 rocksdbjavastaticosx_archs: @@ -2280,7 +2305,7 @@ endif $(MAKE) clean-rocks ARCHFLAG="-arch $*" $(MAKE) rocksdbjavastatic_deps ARCHFLAG="-arch $*" $(MAKE) rocksdbjavastatic_libobjects - ARCHFLAG="-arch $*" ROCKSDBJNILIB="librocksdbjni-osx-$*.jnilib" $(MAKE) rocksdbjavastatic_javalib + ARCHFLAG="-arch $*" FORSTJNILIB="libforstjni-osx-$*.jnilib" $(MAKE) rocksdbjavastatic_javalib ifeq ($(JAR_CMD),) ifneq ($(JAVA_HOME),) @@ -2291,19 +2316,19 @@ endif endif rocksdbjavastatic_javalib: cd java; $(MAKE) javalib - rm -f java/target/$(ROCKSDBJNILIB) + rm -f java/target/$(FORSTJNILIB) $(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \ - -o ./java/target/$(ROCKSDBJNILIB) $(ALL_JNI_NATIVE_SOURCES) \ + -o ./java/target/$(FORSTJNILIB) $(ALL_JNI_NATIVE_SOURCES) \ $(LIB_OBJECTS) $(COVERAGEFLAGS) \ $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS) cd java/target;if [ "$(DEBUG_LEVEL)" == "0" ]; then \ - strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \ + strip $(STRIPFLAGS) $(FORSTJNILIB); \ fi rocksdbjava_jar: cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md - cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) - cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(FORSTJNILIB) + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/forstdb/*.class org/forstdb/util/*.class openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 rocksdbjava_javadocs_jar: @@ -2321,14 +2346,15 @@ rocksdbjavastatic_libobjects: $(LIB_OBJECTS) rocksdbjavastaticrelease: rocksdbjavastaticosx rocksdbjava_javadocs_jar rocksdbjava_sources_jar cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md - cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib - cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) libforstjni-*.so libforstjni-*.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/forstdb/*.class org/forstdb/util/*.class openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1 rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl rocksdbjava_javadocs_jar rocksdbjava_sources_jar cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md - cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib librocksdbjni-win64.dll - cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class + jar -uf java/target/$(ROCKSDB_JAR_ALL) HISTORY*.md + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) libforstjni-*.so libforstjni-*.jnilib libforstjni-win64.dll + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/forstdb/*.class org/forstdb/util/*.class openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1 forstjavastaticreleasedocker: rocksdbjavastaticreleasedocker @@ -2362,6 +2388,7 @@ forstjavastaticreleasedocker: rocksdbjavastaticreleasedocker cd java;cat pom.xml.template | sed 's/\$${FORST_JAVA_VERSION}/$(FORST_JAVA_VERSION)/' > pom.xml cd java;cp pom.xml target/forst-release/$(FJAR_PREF).pom + rocksdbjavastaticdockerx86: mkdir -p java/target docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh @@ -2382,6 +2409,10 @@ rocksdbjavastaticdockers390x: mkdir -p java/target docker run --rm --name rocksdb_linux_s390x-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh +rocksdbjavastaticdockerriscv64: + mkdir -p java/target + docker run --rm --name rocksdb_linux_riscv64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + rocksdbjavastaticdockerx86musl: mkdir -p java/target docker run --rm --name rocksdb_linux_x86-musl-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh @@ -2409,21 +2440,21 @@ rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastatic ROCKSDB_JAVA_RELEASE_CLASSIFIERS = javadoc sources linux64 linux32 linux64-musl linux32-musl osx win64 rocksdbjavastaticpublishcentral: rocksdbjavageneratepom - mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar - $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);) + mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/forstjni-$(ROCKSDB_JAVA_VERSION).jar + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/forstjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);) rocksdbjavageneratepom: cd java;cat pom.xml.template | sed 's/\$${ROCKSDB_JAVA_VERSION}/$(ROCKSDB_JAVA_VERSION)/' > pom.xml rocksdbjavastaticnexusbundlejar: rocksdbjavageneratepom openssl sha1 -r java/pom.xml | awk '{ print $$1 }' > java/target/pom.xml.sha1 - openssl sha1 -r java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar | awk '{ print $$1 }' > java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.sha1 - $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), openssl sha1 -r java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar | awk '{ print $$1 }' > java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.sha1;) + openssl sha1 -r java/target/forstjni-$(ROCKSDB_JAVA_VERSION).jar | awk '{ print $$1 }' > java/target/forstjni-$(ROCKSDB_JAVA_VERSION).jar.sha1 + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), openssl sha1 -r java/target/forstjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar | awk '{ print $$1 }' > java/target/forstjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.sha1;) gpg --yes --output java/target/pom.xml.asc -ab java/pom.xml - gpg --yes -ab java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar - $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), gpg --yes -ab java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar;) - $(JAR_CMD) cvf java/target/nexus-bundle-rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java pom.xml -C java/target pom.xml.sha1 -C java/target pom.xml.asc -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.sha1 -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.asc - $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), $(JAR_CMD) uf java/target/nexus-bundle-rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.sha1 -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.asc;) + gpg --yes -ab java/target/forstjni-$(ROCKSDB_JAVA_VERSION).jar + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), gpg --yes -ab java/target/forstjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar;) + $(JAR_CMD) cvf java/target/nexus-bundle-forstjni-$(ROCKSDB_JAVA_VERSION).jar -C java pom.xml -C java/target pom.xml.sha1 -C java/target pom.xml.asc -C java/target forstjni-$(ROCKSDB_JAVA_VERSION).jar -C java/target forstjni-$(ROCKSDB_JAVA_VERSION).jar.sha1 -C java/target forstjni-$(ROCKSDB_JAVA_VERSION).jar.asc + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), $(JAR_CMD) uf java/target/nexus-bundle-forstjni-$(ROCKSDB_JAVA_VERSION).jar -C java/target forstjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -C java/target forstjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.sha1 -C java/target forstjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.asc;) # A version of each $(LIBOBJECTS) compiled with -fPIC @@ -2436,11 +2467,11 @@ ifeq ($(JAVA_HOME),) $(error JAVA_HOME is not set) endif $(AM_V_GEN)cd java; $(MAKE) javalib; - $(AM_V_at)rm -f ./java/target/$(ROCKSDBJNILIB) - $(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. -I./java/rocksjni $(JAVA_INCLUDE) $(ROCKSDB_PLUGIN_JNI_CXX_INCLUDEFLAGS) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(ALL_JNI_NATIVE_SOURCES) $(LIB_OBJECTS) $(JAVA_LDFLAGS) $(COVERAGEFLAGS) + $(AM_V_at)rm -f ./java/target/$(FORSTJNILIB) + $(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. -I./java/forstjni $(JAVA_INCLUDE) $(ROCKSDB_PLUGIN_JNI_CXX_INCLUDEFLAGS) -shared -fPIC -o ./java/target/$(FORSTJNILIB) $(ALL_JNI_NATIVE_SOURCES) $(LIB_OBJECTS) $(JAVA_LDFLAGS) $(COVERAGEFLAGS) $(AM_V_at)cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md - $(AM_V_at)cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) - $(AM_V_at)cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + $(AM_V_at)cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(FORSTJNILIB) + $(AM_V_at)cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/forstdb/*.class org/forstdb/util/*.class $(AM_V_at)openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 jclean: @@ -2455,6 +2486,9 @@ jtest_run: jtest: rocksdbjava cd java;$(MAKE) sample test +jpmd: rocksdbjava rocksdbjavageneratepom + cd java;$(MAKE) pmd + jdb_bench: cd java;$(MAKE) db_bench; @@ -2474,14 +2508,13 @@ checkout_folly: fi @# Pin to a particular version for public CI, so that PR authors don't @# need to worry about folly breaking our integration. Update periodically - cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba - @# A hack to remove boost dependency. - @# NOTE: this hack is only needed if building using USE_FOLLY_LITE - perl -pi -e 's/^(#include .)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h + @# NOTE: boost source will be needed for any build including `USE_FOLLY_LITE` builds as those depend on boost headers + cd third-party/folly && $(PYTHON) build/fbcode_builder/getdeps.py fetch boost CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS)) @@ -2493,8 +2526,6 @@ build_folly: echo "Please run checkout_folly first"; \ false; \ fi - # Restore the original version of Invoke.h with boost dependency - cd third-party/folly && ${GIT_COMMAND} checkout folly/functional/Invoke.h cd third-party/folly && \ CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests diff --git a/PLUGINS.md b/PLUGINS.md index fefacbede..37fc68b86 100644 --- a/PLUGINS.md +++ b/PLUGINS.md @@ -5,4 +5,5 @@ This is the list of all known third-party plugins for RocksDB. If something is m * [ZenFS](https://github.com/westerndigitalcorporation/zenfs): a file system for zoned block devices * [RADOS](https://github.com/riversand963/rocksdb-rados-env): an Env used for interacting with RADOS. Migrated from RocksDB main repo. * [PMEM](https://github.com/pmem/pmem-rocksdb-plugin): a collection of plugins to enable Persistent Memory on RocksDB. -* [IPPCP](https://github.com/intel/ippcp-plugin-rocksdb): a plugin to enable encryption on RocksDB based on Intel optimized open source IPP-Crypto library. \ No newline at end of file +* [IPPCP](https://github.com/intel/ippcp-plugin-rocksdb): a plugin to enable encryption on RocksDB based on Intel optimized open source IPP-Crypto library. +* [encfs](https://github.com/pegasus-kv/encfs): a plugin to enable encryption on RocksDB based on OpenSSL library. \ No newline at end of file diff --git a/TARGETS b/TARGETS new file mode 100644 index 000000000..afad2672b --- /dev/null +++ b/TARGETS @@ -0,0 +1,5632 @@ +# This file @generated by: +#$ python3 buckifier/buckify_rocksdb.py +# --> DO NOT EDIT MANUALLY <-- +# This file is a Facebook-specific integration for buck builds, so can +# only be validated by Facebook employees. +load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper") + + +cpp_library_wrapper(name="rocksdb_lib", srcs=[ + "cache/cache.cc", + "cache/cache_entry_roles.cc", + "cache/cache_helpers.cc", + "cache/cache_key.cc", + "cache/cache_reservation_manager.cc", + "cache/charged_cache.cc", + "cache/clock_cache.cc", + "cache/compressed_secondary_cache.cc", + "cache/lru_cache.cc", + "cache/secondary_cache.cc", + "cache/secondary_cache_adapter.cc", + "cache/sharded_cache.cc", + "cache/tiered_secondary_cache.cc", + "db/arena_wrapped_db_iter.cc", + "db/blob/blob_contents.cc", + "db/blob/blob_fetcher.cc", + "db/blob/blob_file_addition.cc", + "db/blob/blob_file_builder.cc", + "db/blob/blob_file_cache.cc", + "db/blob/blob_file_garbage.cc", + "db/blob/blob_file_meta.cc", + "db/blob/blob_file_reader.cc", + "db/blob/blob_garbage_meter.cc", + "db/blob/blob_log_format.cc", + "db/blob/blob_log_sequential_reader.cc", + "db/blob/blob_log_writer.cc", + "db/blob/blob_source.cc", + "db/blob/prefetch_buffer_collection.cc", + "db/builder.cc", + "db/c.cc", + "db/column_family.cc", + "db/compaction/compaction.cc", + "db/compaction/compaction_iterator.cc", + "db/compaction/compaction_job.cc", + "db/compaction/compaction_outputs.cc", + "db/compaction/compaction_picker.cc", + "db/compaction/compaction_picker_fifo.cc", + "db/compaction/compaction_picker_level.cc", + "db/compaction/compaction_picker_universal.cc", + "db/compaction/compaction_service_job.cc", + "db/compaction/compaction_state.cc", + "db/compaction/sst_partitioner.cc", + "db/compaction/subcompaction_state.cc", + "db/convenience.cc", + "db/db_filesnapshot.cc", + "db/db_impl/compacted_db_impl.cc", + "db/db_impl/db_impl.cc", + "db/db_impl/db_impl_compaction_flush.cc", + "db/db_impl/db_impl_debug.cc", + "db/db_impl/db_impl_experimental.cc", + "db/db_impl/db_impl_files.cc", + "db/db_impl/db_impl_open.cc", + "db/db_impl/db_impl_readonly.cc", + "db/db_impl/db_impl_secondary.cc", + "db/db_impl/db_impl_write.cc", + "db/db_info_dumper.cc", + "db/db_iter.cc", + "db/dbformat.cc", + "db/error_handler.cc", + "db/event_helpers.cc", + "db/experimental.cc", + "db/external_sst_file_ingestion_job.cc", + "db/file_indexer.cc", + "db/flush_job.cc", + "db/flush_scheduler.cc", + "db/forward_iterator.cc", + "db/import_column_family_job.cc", + "db/internal_stats.cc", + "db/log_reader.cc", + "db/log_writer.cc", + "db/logs_with_prep_tracker.cc", + "db/malloc_stats.cc", + "db/memtable.cc", + "db/memtable_list.cc", + "db/merge_helper.cc", + "db/merge_operator.cc", + "db/output_validator.cc", + "db/periodic_task_scheduler.cc", + "db/range_del_aggregator.cc", + "db/range_tombstone_fragmenter.cc", + "db/repair.cc", + "db/seqno_to_time_mapping.cc", + "db/snapshot_impl.cc", + "db/table_cache.cc", + "db/table_properties_collector.cc", + "db/transaction_log_impl.cc", + "db/trim_history_scheduler.cc", + "db/version_builder.cc", + "db/version_edit.cc", + "db/version_edit_handler.cc", + "db/version_set.cc", + "db/wal_edit.cc", + "db/wal_manager.cc", + "db/wide/wide_column_serialization.cc", + "db/wide/wide_columns.cc", + "db/wide/wide_columns_helper.cc", + "db/write_batch.cc", + "db/write_batch_base.cc", + "db/write_controller.cc", + "db/write_stall_stats.cc", + "db/write_thread.cc", + "env/composite_env.cc", + "env/env.cc", + "env/env_chroot.cc", + "env/env_encryption.cc", + "env/env_posix.cc", + "env/file_system.cc", + "env/file_system_tracer.cc", + "env/fs_posix.cc", + "env/fs_remap.cc", + "env/io_posix.cc", + "env/mock_env.cc", + "env/unique_id_gen.cc", + "file/delete_scheduler.cc", + "file/file_prefetch_buffer.cc", + "file/file_util.cc", + "file/filename.cc", + "file/line_file_reader.cc", + "file/random_access_file_reader.cc", + "file/read_write_util.cc", + "file/readahead_raf.cc", + "file/sequence_file_reader.cc", + "file/sst_file_manager_impl.cc", + "file/writable_file_writer.cc", + "logging/auto_roll_logger.cc", + "logging/event_logger.cc", + "logging/log_buffer.cc", + "memory/arena.cc", + "memory/concurrent_arena.cc", + "memory/jemalloc_nodump_allocator.cc", + "memory/memkind_kmem_allocator.cc", + "memory/memory_allocator.cc", + "memtable/alloc_tracker.cc", + "memtable/hash_linklist_rep.cc", + "memtable/hash_skiplist_rep.cc", + "memtable/skiplistrep.cc", + "memtable/vectorrep.cc", + "memtable/write_buffer_manager.cc", + "monitoring/histogram.cc", + "monitoring/histogram_windowing.cc", + "monitoring/in_memory_stats_history.cc", + "monitoring/instrumented_mutex.cc", + "monitoring/iostats_context.cc", + "monitoring/perf_context.cc", + "monitoring/perf_level.cc", + "monitoring/persistent_stats_history.cc", + "monitoring/statistics.cc", + "monitoring/thread_status_impl.cc", + "monitoring/thread_status_updater.cc", + "monitoring/thread_status_updater_debug.cc", + "monitoring/thread_status_util.cc", + "monitoring/thread_status_util_debug.cc", + "options/cf_options.cc", + "options/configurable.cc", + "options/customizable.cc", + "options/db_options.cc", + "options/offpeak_time_info.cc", + "options/options.cc", + "options/options_helper.cc", + "options/options_parser.cc", + "port/mmap.cc", + "port/port_posix.cc", + "port/stack_trace.cc", + "port/win/env_default.cc", + "port/win/env_win.cc", + "port/win/io_win.cc", + "port/win/port_win.cc", + "port/win/win_logger.cc", + "port/win/win_thread.cc", + "table/adaptive/adaptive_table_factory.cc", + "table/block_based/binary_search_index_reader.cc", + "table/block_based/block.cc", + "table/block_based/block_based_table_builder.cc", + "table/block_based/block_based_table_factory.cc", + "table/block_based/block_based_table_iterator.cc", + "table/block_based/block_based_table_reader.cc", + "table/block_based/block_builder.cc", + "table/block_based/block_cache.cc", + "table/block_based/block_prefetcher.cc", + "table/block_based/block_prefix_index.cc", + "table/block_based/data_block_footer.cc", + "table/block_based/data_block_hash_index.cc", + "table/block_based/filter_block_reader_common.cc", + "table/block_based/filter_policy.cc", + "table/block_based/flush_block_policy.cc", + "table/block_based/full_filter_block.cc", + "table/block_based/hash_index_reader.cc", + "table/block_based/index_builder.cc", + "table/block_based/index_reader_common.cc", + "table/block_based/parsed_full_filter_block.cc", + "table/block_based/partitioned_filter_block.cc", + "table/block_based/partitioned_index_iterator.cc", + "table/block_based/partitioned_index_reader.cc", + "table/block_based/reader_common.cc", + "table/block_based/uncompression_dict_reader.cc", + "table/block_fetcher.cc", + "table/compaction_merging_iterator.cc", + "table/cuckoo/cuckoo_table_builder.cc", + "table/cuckoo/cuckoo_table_factory.cc", + "table/cuckoo/cuckoo_table_reader.cc", + "table/format.cc", + "table/get_context.cc", + "table/iterator.cc", + "table/merging_iterator.cc", + "table/meta_blocks.cc", + "table/persistent_cache_helper.cc", + "table/plain/plain_table_bloom.cc", + "table/plain/plain_table_builder.cc", + "table/plain/plain_table_factory.cc", + "table/plain/plain_table_index.cc", + "table/plain/plain_table_key_coding.cc", + "table/plain/plain_table_reader.cc", + "table/sst_file_dumper.cc", + "table/sst_file_reader.cc", + "table/sst_file_writer.cc", + "table/table_factory.cc", + "table/table_properties.cc", + "table/two_level_iterator.cc", + "table/unique_id.cc", + "test_util/sync_point.cc", + "test_util/sync_point_impl.cc", + "test_util/transaction_test_util.cc", + "tools/dump/db_dump_tool.cc", + "tools/io_tracer_parser_tool.cc", + "tools/ldb_cmd.cc", + "tools/ldb_tool.cc", + "tools/sst_dump_tool.cc", + "trace_replay/block_cache_tracer.cc", + "trace_replay/io_tracer.cc", + "trace_replay/trace_record.cc", + "trace_replay/trace_record_handler.cc", + "trace_replay/trace_record_result.cc", + "trace_replay/trace_replay.cc", + "util/async_file_reader.cc", + "util/build_version.cc", + "util/cleanable.cc", + "util/coding.cc", + "util/compaction_job_stats_impl.cc", + "util/comparator.cc", + "util/compression.cc", + "util/compression_context_cache.cc", + "util/concurrent_task_limiter_impl.cc", + "util/crc32c.cc", + "util/crc32c_arm64.cc", + "util/data_structure.cc", + "util/dynamic_bloom.cc", + "util/file_checksum_helper.cc", + "util/hash.cc", + "util/murmurhash.cc", + "util/random.cc", + "util/rate_limiter.cc", + "util/ribbon_config.cc", + "util/slice.cc", + "util/status.cc", + "util/stderr_logger.cc", + "util/string_util.cc", + "util/thread_local.cc", + "util/threadpool_imp.cc", + "util/udt_util.cc", + "util/write_batch_util.cc", + "util/xxhash.cc", + "utilities/agg_merge/agg_merge.cc", + "utilities/backup/backup_engine.cc", + "utilities/blob_db/blob_compaction_filter.cc", + "utilities/blob_db/blob_db.cc", + "utilities/blob_db/blob_db_impl.cc", + "utilities/blob_db/blob_db_impl_filesnapshot.cc", + "utilities/blob_db/blob_dump_tool.cc", + "utilities/blob_db/blob_file.cc", + "utilities/cache_dump_load.cc", + "utilities/cache_dump_load_impl.cc", + "utilities/cassandra/cassandra_compaction_filter.cc", + "utilities/cassandra/format.cc", + "utilities/cassandra/merge_operator.cc", + "utilities/checkpoint/checkpoint_impl.cc", + "utilities/compaction_filters.cc", + "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc", + "utilities/convenience/info_log_finder.cc", + "utilities/counted_fs.cc", + "utilities/debug.cc", + "utilities/env_mirror.cc", + "utilities/env_timed.cc", + "utilities/fault_injection_env.cc", + "utilities/fault_injection_fs.cc", + "utilities/fault_injection_secondary_cache.cc", + "utilities/flink/flink_compaction_filter.cc", + "utilities/leveldb_options/leveldb_options.cc", + "utilities/memory/memory_util.cc", + "utilities/merge_operators.cc", + "utilities/merge_operators/bytesxor.cc", + "utilities/merge_operators/max.cc", + "utilities/merge_operators/put.cc", + "utilities/merge_operators/sortlist.cc", + "utilities/merge_operators/string_append/stringappend.cc", + "utilities/merge_operators/string_append/stringappend2.cc", + "utilities/merge_operators/uint64add.cc", + "utilities/object_registry.cc", + "utilities/option_change_migration/option_change_migration.cc", + "utilities/options/options_util.cc", + "utilities/persistent_cache/block_cache_tier.cc", + "utilities/persistent_cache/block_cache_tier_file.cc", + "utilities/persistent_cache/block_cache_tier_metadata.cc", + "utilities/persistent_cache/persistent_cache_tier.cc", + "utilities/persistent_cache/volatile_tier_impl.cc", + "utilities/simulator_cache/cache_simulator.cc", + "utilities/simulator_cache/sim_cache.cc", + "utilities/table_properties_collectors/compact_on_deletion_collector.cc", + "utilities/trace/file_trace_reader_writer.cc", + "utilities/trace/replayer_impl.cc", + "utilities/transactions/lock/lock_manager.cc", + "utilities/transactions/lock/point/point_lock_manager.cc", + "utilities/transactions/lock/point/point_lock_tracker.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc", + "utilities/transactions/lock/range/range_tree/lib/standalone_port.cc", + "utilities/transactions/lock/range/range_tree/lib/util/dbt.cc", + "utilities/transactions/lock/range/range_tree/lib/util/memarena.cc", + "utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc", + "utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc", + "utilities/transactions/optimistic_transaction.cc", + "utilities/transactions/optimistic_transaction_db_impl.cc", + "utilities/transactions/pessimistic_transaction.cc", + "utilities/transactions/pessimistic_transaction_db.cc", + "utilities/transactions/snapshot_checker.cc", + "utilities/transactions/transaction_base.cc", + "utilities/transactions/transaction_db_mutex_impl.cc", + "utilities/transactions/transaction_util.cc", + "utilities/transactions/write_prepared_txn.cc", + "utilities/transactions/write_prepared_txn_db.cc", + "utilities/transactions/write_unprepared_txn.cc", + "utilities/transactions/write_unprepared_txn_db.cc", + "utilities/ttl/db_ttl_impl.cc", + "utilities/wal_filter.cc", + "utilities/write_batch_with_index/write_batch_with_index.cc", + "utilities/write_batch_with_index/write_batch_with_index_internal.cc", + ], deps=[ + "//folly/container:f14_hash", + "//folly/experimental/coro:blocking_wait", + "//folly/experimental/coro:collect", + "//folly/experimental/coro:coroutine", + "//folly/experimental/coro:task", + "//folly/synchronization:distributed_mutex", + ], headers=None, link_whole=False, extra_test_libs=False) + +cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[], deps=[":rocksdb_lib"], headers=None, link_whole=True, extra_test_libs=False) + +cpp_library_wrapper(name="rocksdb_test_lib", srcs=[ + "db/db_test_util.cc", + "db/db_with_timestamp_test_util.cc", + "table/mock_table.cc", + "test_util/mock_time_env.cc", + "test_util/secondary_cache_test_util.cc", + "test_util/testharness.cc", + "test_util/testutil.cc", + "tools/block_cache_analyzer/block_cache_trace_analyzer.cc", + "tools/trace_analyzer_tool.cc", + "utilities/agg_merge/test_agg_merge.cc", + "utilities/cassandra/test_utils.cc", + ], deps=[":rocksdb_lib"], headers=None, link_whole=False, extra_test_libs=True) + +cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[ + "test_util/testutil.cc", + "tools/block_cache_analyzer/block_cache_trace_analyzer.cc", + "tools/db_bench_tool.cc", + "tools/simulated_hybrid_file_system.cc", + "tools/trace_analyzer_tool.cc", + ], deps=[":rocksdb_lib"], headers=None, link_whole=False, extra_test_libs=False) + +cpp_library_wrapper(name="rocksdb_cache_bench_tools_lib", srcs=["cache/cache_bench_tool.cc"], deps=[":rocksdb_lib"], headers=None, link_whole=False, extra_test_libs=False) + +rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[ + "db_stress_tool/batched_ops_stress.cc", + "db_stress_tool/cf_consistency_stress.cc", + "db_stress_tool/db_stress_common.cc", + "db_stress_tool/db_stress_driver.cc", + "db_stress_tool/db_stress_gflags.cc", + "db_stress_tool/db_stress_listener.cc", + "db_stress_tool/db_stress_shared_state.cc", + "db_stress_tool/db_stress_stat.cc", + "db_stress_tool/db_stress_test_base.cc", + "db_stress_tool/db_stress_tool.cc", + "db_stress_tool/db_stress_wide_merge_operator.cc", + "db_stress_tool/expected_state.cc", + "db_stress_tool/expected_value.cc", + "db_stress_tool/multi_ops_txns_stress.cc", + "db_stress_tool/no_batched_ops_stress.cc", + "test_util/testutil.cc", + "tools/block_cache_analyzer/block_cache_trace_analyzer.cc", + "tools/trace_analyzer_tool.cc", + ], headers=None) + + +cpp_binary_wrapper(name="db_stress", srcs=["db_stress_tool/db_stress.cc"], deps=[":rocksdb_stress_lib"], extra_preprocessor_flags=[], extra_bench_libs=False) + +cpp_binary_wrapper(name="cache_bench", srcs=["cache/cache_bench.cc"], deps=[":rocksdb_cache_bench_tools_lib"], extra_preprocessor_flags=[], extra_bench_libs=False) + +cpp_binary_wrapper(name="ribbon_bench", srcs=["microbench/ribbon_bench.cc"], deps=[], extra_preprocessor_flags=[], extra_bench_libs=True) + +cpp_binary_wrapper(name="db_basic_bench", srcs=["microbench/db_basic_bench.cc"], deps=[], extra_preprocessor_flags=[], extra_bench_libs=True) + +add_c_test_wrapper() + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_0", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2438, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_1", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_2", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}}, slow=False, expected_runtime=2446, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_3", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DataBlockSeek/iterations:1000000': ['real_time', + 'cpu_time', + 'seek_ns', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_4", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'RandomAccessFileReaderRead/enable_statistics:1/iterations:1000000': ['real_time', + 'cpu_time', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_5", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_6", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_7", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'RandomAccessFileReaderRead/enable_statistics:0/iterations:1000000': ['real_time', + 'cpu_time', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct']}}, slow=False, expected_runtime=2438, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_8", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_9", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_10", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_11", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}}, slow=False, expected_runtime=2446, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_12", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_13", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_14", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_0_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88891, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_1_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88804, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_2_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88803, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_3_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88891, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_4_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88809, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_5_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88803, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_6_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88813, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_7_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88813, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_8_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88709, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_9_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88711, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_10_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88819, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_11_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88711, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_12_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88709, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_13_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88709, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_14_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88711, sl_iterations=3, regression_threshold=10) + + + # Generate a test rule for each entry in ROCKS_TESTS + # Do not build the tests in opt mode, since SyncPoint and other test code + # will not be included. + +cpp_unittest_wrapper(name="agg_merge_test", + srcs=["utilities/agg_merge/agg_merge_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="arena_test", + srcs=["memory/arena_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="auto_roll_logger_test", + srcs=["logging/auto_roll_logger_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="autovector_test", + srcs=["util/autovector_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="backup_engine_test", + srcs=["utilities/backup/backup_engine_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_counting_iterator_test", + srcs=["db/blob/blob_counting_iterator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_db_test", + srcs=["utilities/blob_db/blob_db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_addition_test", + srcs=["db/blob/blob_file_addition_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_builder_test", + srcs=["db/blob/blob_file_builder_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_cache_test", + srcs=["db/blob/blob_file_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_garbage_test", + srcs=["db/blob/blob_file_garbage_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_reader_test", + srcs=["db/blob/blob_file_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_garbage_meter_test", + srcs=["db/blob/blob_garbage_meter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_source_test", + srcs=["db/blob/blob_source_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_based_table_reader_test", + srcs=["table/block_based/block_based_table_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_cache_trace_analyzer_test", + srcs=["tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_cache_tracer_test", + srcs=["trace_replay/block_cache_tracer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_fetcher_test", + srcs=["table/block_fetcher_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_test", + srcs=["table/block_based/block_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="bloom_test", + srcs=["util/bloom_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cache_reservation_manager_test", + srcs=["cache/cache_reservation_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cache_simulator_test", + srcs=["utilities/simulator_cache/cache_simulator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cache_test", + srcs=["cache/cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cassandra_format_test", + srcs=["utilities/cassandra/cassandra_format_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cassandra_functional_test", + srcs=["utilities/cassandra/cassandra_functional_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cassandra_row_merge_test", + srcs=["utilities/cassandra/cassandra_row_merge_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cassandra_serialize_test", + srcs=["utilities/cassandra/cassandra_serialize_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="checkpoint_test", + srcs=["utilities/checkpoint/checkpoint_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cleanable_test", + srcs=["table/cleanable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="clipping_iterator_test", + srcs=["db/compaction/clipping_iterator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="coding_test", + srcs=["util/coding_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="column_family_test", + srcs=["db/column_family_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compact_files_test", + srcs=["db/compact_files_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compact_on_deletion_collector_test", + srcs=["utilities/table_properties_collectors/compact_on_deletion_collector_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_iterator_test", + srcs=["db/compaction/compaction_iterator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_job_stats_test", + srcs=["db/compaction/compaction_job_stats_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_job_test", + srcs=["db/compaction/compaction_job_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_picker_test", + srcs=["db/compaction/compaction_picker_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_service_test", + srcs=["db/compaction/compaction_service_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="comparator_db_test", + srcs=["db/comparator_db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compressed_secondary_cache_test", + srcs=["cache/compressed_secondary_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="configurable_test", + srcs=["options/configurable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="corruption_test", + srcs=["db/corruption_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="crc32c_test", + srcs=["util/crc32c_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cuckoo_table_builder_test", + srcs=["table/cuckoo/cuckoo_table_builder_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cuckoo_table_db_test", + srcs=["db/cuckoo_table_db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cuckoo_table_reader_test", + srcs=["table/cuckoo/cuckoo_table_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="customizable_test", + srcs=["options/customizable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="data_block_hash_index_test", + srcs=["table/block_based/data_block_hash_index_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_basic_test", + srcs=["db/db_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_blob_basic_test", + srcs=["db/blob/db_blob_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_blob_compaction_test", + srcs=["db/blob/db_blob_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_blob_corruption_test", + srcs=["db/blob/db_blob_corruption_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_blob_index_test", + srcs=["db/blob/db_blob_index_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_block_cache_test", + srcs=["db/db_block_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_bloom_filter_test", + srcs=["db/db_bloom_filter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_clip_test", + srcs=["db/db_clip_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_compaction_filter_test", + srcs=["db/db_compaction_filter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_compaction_test", + srcs=["db/db_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_dynamic_level_test", + srcs=["db/db_dynamic_level_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_encryption_test", + srcs=["db/db_encryption_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_flush_test", + srcs=["db/db_flush_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_inplace_update_test", + srcs=["db/db_inplace_update_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_io_failure_test", + srcs=["db/db_io_failure_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_iter_stress_test", + srcs=["db/db_iter_stress_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_iter_test", + srcs=["db/db_iter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_iterator_test", + srcs=["db/db_iterator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_kv_checksum_test", + srcs=["db/db_kv_checksum_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_log_iter_test", + srcs=["db/db_log_iter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_logical_block_size_cache_test", + srcs=["db/db_logical_block_size_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_memtable_test", + srcs=["db/db_memtable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_merge_operand_test", + srcs=["db/db_merge_operand_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_merge_operator_test", + srcs=["db/db_merge_operator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_options_test", + srcs=["db/db_options_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_properties_test", + srcs=["db/db_properties_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_range_del_test", + srcs=["db/db_range_del_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_rate_limiter_test", + srcs=["db/db_rate_limiter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_readonly_with_timestamp_test", + srcs=["db/db_readonly_with_timestamp_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_secondary_test", + srcs=["db/db_secondary_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_sst_test", + srcs=["db/db_sst_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_statistics_test", + srcs=["db/db_statistics_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_table_properties_test", + srcs=["db/db_table_properties_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_tailing_iter_test", + srcs=["db/db_tailing_iter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_test", + srcs=["db/db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_test2", + srcs=["db/db_test2.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_universal_compaction_test", + srcs=["db/db_universal_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_wal_test", + srcs=["db/db_wal_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_wide_basic_test", + srcs=["db/wide/db_wide_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_with_timestamp_basic_test", + srcs=["db/db_with_timestamp_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_with_timestamp_compaction_test", + srcs=["db/db_with_timestamp_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_write_buffer_manager_test", + srcs=["db/db_write_buffer_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_write_test", + srcs=["db/db_write_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="dbformat_test", + srcs=["db/dbformat_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="defer_test", + srcs=["util/defer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="delete_scheduler_test", + srcs=["file/delete_scheduler_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="deletefile_test", + srcs=["db/deletefile_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="dynamic_bloom_test", + srcs=["util/dynamic_bloom_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_library_wrapper(name="env_basic_test_lib", srcs=["env/env_basic_test.cc"], deps=[":rocksdb_test_lib"], headers=None, link_whole=False, extra_test_libs=True) + +cpp_unittest_wrapper(name="env_basic_test", + srcs=["env/env_basic_test.cc"], + deps=[":env_basic_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="env_logger_test", + srcs=["logging/env_logger_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="env_test", + srcs=["env/env_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="env_timed_test", + srcs=["utilities/env_timed_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="error_handler_fs_test", + srcs=["db/error_handler_fs_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="event_logger_test", + srcs=["logging/event_logger_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="external_sst_file_basic_test", + srcs=["db/external_sst_file_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="external_sst_file_test", + srcs=["db/external_sst_file_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="fault_injection_test", + srcs=["db/fault_injection_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="file_indexer_test", + srcs=["db/file_indexer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="file_reader_writer_test", + srcs=["util/file_reader_writer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="filelock_test", + srcs=["util/filelock_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="filename_test", + srcs=["db/filename_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="flink_compaction_filter_test", + srcs=["utilities/flink/flink_compaction_filter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="flush_job_test", + srcs=["db/flush_job_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="full_filter_block_test", + srcs=["table/block_based/full_filter_block_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="hash_table_test", + srcs=["utilities/persistent_cache/hash_table_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="hash_test", + srcs=["util/hash_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="heap_test", + srcs=["util/heap_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="histogram_test", + srcs=["monitoring/histogram_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="import_column_family_test", + srcs=["db/import_column_family_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="inlineskiplist_test", + srcs=["memtable/inlineskiplist_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="io_posix_test", + srcs=["env/io_posix_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="io_tracer_parser_test", + srcs=["tools/io_tracer_parser_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="io_tracer_test", + srcs=["trace_replay/io_tracer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="iostats_context_test", + srcs=["monitoring/iostats_context_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="ldb_cmd_test", + srcs=["tools/ldb_cmd_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="listener_test", + srcs=["db/listener_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="log_test", + srcs=["db/log_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="lru_cache_test", + srcs=["cache/lru_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="manual_compaction_test", + srcs=["db/manual_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="memory_allocator_test", + srcs=["memory/memory_allocator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="memory_test", + srcs=["utilities/memory/memory_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="memtable_list_test", + srcs=["db/memtable_list_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="merge_helper_test", + srcs=["db/merge_helper_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="merge_test", + srcs=["db/merge_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="merger_test", + srcs=["table/merger_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="mock_env_test", + srcs=["env/mock_env_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="object_registry_test", + srcs=["utilities/object_registry_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="obsolete_files_test", + srcs=["db/obsolete_files_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="optimistic_transaction_test", + srcs=["utilities/transactions/optimistic_transaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="option_change_migration_test", + srcs=["utilities/option_change_migration/option_change_migration_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="options_file_test", + srcs=["db/options_file_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="options_settable_test", + srcs=["options/options_settable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="options_test", + srcs=["options/options_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="options_util_test", + srcs=["utilities/options/options_util_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="partitioned_filter_block_test", + srcs=["table/block_based/partitioned_filter_block_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="perf_context_test", + srcs=["db/perf_context_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="periodic_task_scheduler_test", + srcs=["db/periodic_task_scheduler_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="persistent_cache_test", + srcs=["utilities/persistent_cache/persistent_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="plain_table_db_test", + srcs=["db/plain_table_db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="point_lock_manager_test", + srcs=["utilities/transactions/lock/point/point_lock_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="prefetch_test", + srcs=["file/prefetch_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="prefix_test", + srcs=["db/prefix_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="random_access_file_reader_test", + srcs=["file/random_access_file_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="random_test", + srcs=["util/random_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="range_del_aggregator_test", + srcs=["db/range_del_aggregator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="range_locking_test", + srcs=["utilities/transactions/lock/range/range_locking_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="range_tombstone_fragmenter_test", + srcs=["db/range_tombstone_fragmenter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="rate_limiter_test", + srcs=["util/rate_limiter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="reduce_levels_test", + srcs=["tools/reduce_levels_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="repair_test", + srcs=["db/repair_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="repeatable_thread_test", + srcs=["util/repeatable_thread_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="ribbon_test", + srcs=["util/ribbon_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="seqno_time_test", + srcs=["db/seqno_time_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="sim_cache_test", + srcs=["utilities/simulator_cache/sim_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="skiplist_test", + srcs=["memtable/skiplist_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="slice_test", + srcs=["util/slice_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="slice_transform_test", + srcs=["util/slice_transform_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="sst_dump_test", + srcs=["tools/sst_dump_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="sst_file_reader_test", + srcs=["table/sst_file_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="statistics_test", + srcs=["monitoring/statistics_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="stats_history_test", + srcs=["monitoring/stats_history_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="stringappend_test", + srcs=["utilities/merge_operators/string_append/stringappend_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="table_properties_collector_test", + srcs=["db/table_properties_collector_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="table_test", + srcs=["table/table_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="testutil_test", + srcs=["test_util/testutil_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="thread_list_test", + srcs=["util/thread_list_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="thread_local_test", + srcs=["util/thread_local_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="tiered_compaction_test", + srcs=["db/compaction/tiered_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="tiered_secondary_cache_test", + srcs=["cache/tiered_secondary_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="timer_queue_test", + srcs=["util/timer_queue_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="timer_test", + srcs=["util/timer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="timestamped_snapshot_test", + srcs=["utilities/transactions/timestamped_snapshot_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="trace_analyzer_test", + srcs=["tools/trace_analyzer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="transaction_test", + srcs=["utilities/transactions/transaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="ttl_test", + srcs=["utilities/ttl/ttl_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="udt_util_test", + srcs=["util/udt_util_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="util_merge_operators_test", + srcs=["utilities/util_merge_operators_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="version_builder_test", + srcs=["db/version_builder_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="version_edit_test", + srcs=["db/version_edit_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="version_set_test", + srcs=["db/version_set_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="wal_manager_test", + srcs=["db/wal_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="wide_column_serialization_test", + srcs=["db/wide/wide_column_serialization_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="wide_columns_helper_test", + srcs=["db/wide/wide_columns_helper_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="work_queue_test", + srcs=["util/work_queue_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_batch_test", + srcs=["db/write_batch_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_batch_with_index_test", + srcs=["utilities/write_batch_with_index/write_batch_with_index_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_buffer_manager_test", + srcs=["memtable/write_buffer_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_callback_test", + srcs=["db/write_callback_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_committed_transaction_ts_test", + srcs=["utilities/transactions/write_committed_transaction_ts_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_controller_test", + srcs=["db/write_controller_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_prepared_transaction_test", + srcs=["utilities/transactions/write_prepared_transaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_unprepared_transaction_test", + srcs=["utilities/transactions/write_unprepared_transaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +export_file(name = "tools/db_crashtest.py") diff --git a/USERS.md b/USERS.md index 43c9cac1e..086cab90d 100644 --- a/USERS.md +++ b/USERS.md @@ -152,6 +152,9 @@ LzLabs is using RocksDB as a storage engine in their multi-database distributed ## ArangoDB [ArangoDB](https://www.arangodb.com/) is a native multi-model database with flexible data models for documents, graphs, and key-values, for building high performance applications using a convenient SQL-like query language or JavaScript extensions. It uses RocksDB as its storage engine. +## Qdrant +[Qdrant](https://qdrant.tech/) is an open source vector database, it [uses](https://qdrant.tech/documentation/concepts/storage/) RocksDB as its persistent storage. + ## Milvus [Milvus](https://milvus.io/) is an open source vector database for unstructured data. It uses RocksDB not only as one of the supported kv storage engines, but also as a message queue. @@ -161,5 +164,9 @@ LzLabs is using RocksDB as a storage engine in their multi-database distributed ## Solana Labs [Solana](https://github.com/solana-labs/solana) is a fast, secure, scalable, and decentralized blockchain. It uses RocksDB as the underlying storage for its ledger store. +## Apache Kvrocks + +[Apache Kvrocks](https://github.com/apache/kvrocks) is an open-source distributed key-value NoSQL database built on top of RocksDB. It serves as a cost-saving and capacity-increasing alternative drop-in replacement for Redis. + ## Others More databases using RocksDB can be found at [dbdb.io](https://dbdb.io/browse?embeds=rocksdb). diff --git a/buckifier/buckify_rocksdb.py b/buckifier/buckify_rocksdb.py index a9e7b447d..b56e1a82d 100755 --- a/buckifier/buckify_rocksdb.py +++ b/buckifier/buckify_rocksdb.py @@ -197,6 +197,10 @@ def generate_targets(repo_path, deps_map): TARGETS.add_binary( "db_stress", ["db_stress_tool/db_stress.cc"], [":rocksdb_stress_lib"] ) + # cache_bench binary + TARGETS.add_binary( + "cache_bench", ["cache/cache_bench.cc"], [":rocksdb_cache_bench_tools_lib"] + ) # bench binaries for src in src_mk.get("MICROBENCH_SOURCES", []): name = src.rsplit("/", 1)[1].split(".")[0] if "/" in src else src.split(".")[0] @@ -299,6 +303,7 @@ def generate_targets(repo_path, deps_map): deps=json.dumps(deps["extra_deps"] + [":rocksdb_test_lib"]), extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]), ) + TARGETS.export_file("tools/db_crashtest.py") print(ColorString.info("Generated TARGETS Summary:")) print(ColorString.info("- %d libs" % TARGETS.total_lib)) diff --git a/buckifier/targets_builder.py b/buckifier/targets_builder.py index 343b2207d..f5d727469 100644 --- a/buckifier/targets_builder.py +++ b/buckifier/targets_builder.py @@ -23,7 +23,7 @@ def pretty_list(lst, indent=8): return res -class TARGETSBuilder(object): +class TARGETSBuilder: def __init__(self, path, extra_argv): self.path = path header = targets_cfg.rocksdb_target_header_template.format( @@ -148,3 +148,9 @@ def register_test(self, test_name, src, deps, extra_compiler_flags): ).encode("utf-8") ) self.total_test = self.total_test + 1 + + def export_file(self, name): + with open(self.path, "a") as targets_file: + targets_file.write( + targets_cfg.export_file_template.format(name=name) + ) diff --git a/buckifier/targets_cfg.py b/buckifier/targets_cfg.py index 491c34d6e..ead6ac51a 100644 --- a/buckifier/targets_cfg.py +++ b/buckifier/targets_cfg.py @@ -6,8 +6,6 @@ # --> DO NOT EDIT MANUALLY <-- # This file is a Facebook-specific integration for buck builds, so can # only be validated by Facebook employees. -# -# @noautodeps @nocodemods load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper") """ @@ -39,3 +37,7 @@ fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime}, sl_iterations={sl_iterations}, regression_threshold={regression_threshold}) """ + +export_file_template = """ +export_file(name = "{name}") +""" diff --git a/buckifier/util.py b/buckifier/util.py index 8943fed2b..be197efd0 100644 --- a/buckifier/util.py +++ b/buckifier/util.py @@ -14,7 +14,7 @@ import time -class ColorString(object): +class ColorString: """Generate colorful strings on terminal""" HEADER = "\033[95m" diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index f7e9c866b..87932ba7a 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -603,7 +603,7 @@ EOF PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark" fi fi - if test $USE_FOLLY; then + if test $USE_FOLLY || test $USE_FOLLY_LITE; then # Test whether libfolly library is installed $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < @@ -647,8 +647,10 @@ if [ "$PORTABLE" == "" ] || [ "$PORTABLE" == 0 ]; then fi COMMON_FLAGS="$COMMON_FLAGS" elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then - RISC_ISA=$(cat /proc/cpuinfo | grep isa | head -1 | cut --delimiter=: -f 2 | cut -b 2-) - COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}" + RISC_ISA=$(cat /proc/cpuinfo | grep -E '^isa\s*:' | head -1 | cut --delimiter=: -f 2 | cut -b 2-) + if [ -n "${RISCV_ISA}" ]; then + COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}" + fi elif [ "$TARGET_OS" == "IOS" ]; then COMMON_FLAGS="$COMMON_FLAGS" else @@ -660,8 +662,7 @@ else if test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then COMMON_FLAGS="$COMMON_FLAGS -march=z196 " elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then - RISC_ISA=$(cat /proc/cpuinfo | grep isa | head -1 | cut --delimiter=: -f 2 | cut -b 2-) - COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}" + COMMON_FLAGS="$COMMON_FLAGS -march=rv64gc" elif test "$USE_SSE"; then # USE_SSE is DEPRECATED # This is a rough approximation of the old USE_SSE behavior @@ -674,13 +675,13 @@ else fi if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then - # For portability compile for macOS 10.13 (2017) or newer - COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.13" - PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.13" + # For portability compile for macOS 10.14 (2018) or newer + COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.14" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.14" # -mmacosx-version-min must come first here. - PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.13 $PLATFORM_SHARED_LDFLAGS" - PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" - JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.13" + PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.14 $PLATFORM_SHARED_LDFLAGS" + PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.14" + JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.14" JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" JAVA_STATIC_DEPS_CCFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" JAVA_STATIC_DEPS_CXXFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" @@ -750,6 +751,11 @@ if [ "$USE_FOLLY" ]; then FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly` fi fi +if [ "$USE_FOLLY_LITE" ]; then + if [ "$FOLLY_DIR" ]; then + BOOST_SOURCE_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-source-dir boost` + fi +fi PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" @@ -791,6 +797,7 @@ echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT" echo "FIND=$FIND" >> "$OUTPUT" echo "WATCH=$WATCH" >> "$OUTPUT" echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT" +echo "BOOST_SOURCE_PATH=$BOOST_SOURCE_PATH" >> "$OUTPUT" # This will enable some related identifiers for the preprocessor if test -n "$JEMALLOC"; then diff --git a/build_tools/error_filter.py b/build_tools/error_filter.py index c42df1f91..d9cb1099c 100644 --- a/build_tools/error_filter.py +++ b/build_tools/error_filter.py @@ -15,7 +15,7 @@ import sys -class ErrorParserBase(object): +class ErrorParserBase: def parse_error(self, line): """Parses a line of test output. If it contains an error, returns a formatted message describing the error; otherwise, returns None. diff --git a/cache/cache.cc b/cache/cache.cc index a65f5ec4f..3dbea128e 100644 --- a/cache/cache.cc +++ b/cache/cache.cc @@ -66,6 +66,41 @@ static std::unordered_map OptionTypeFlags::kMutable}}, }; +namespace { +static void NoopDelete(Cache::ObjectPtr /*obj*/, + MemoryAllocator* /*allocator*/) { + assert(false); +} + +static size_t SliceSize(Cache::ObjectPtr obj) { + return static_cast(obj)->size(); +} + +static Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset, + size_t length, char* out) { + const Slice& slice = *static_cast(from_obj); + std::memcpy(out, slice.data() + from_offset, length); + return Status::OK(); +} + +static Status NoopCreate(const Slice& /*data*/, CompressionType /*type*/, + CacheTier /*source*/, Cache::CreateContext* /*ctx*/, + MemoryAllocator* /*allocator*/, + Cache::ObjectPtr* /*out_obj*/, + size_t* /*out_charge*/) { + assert(false); + return Status::NotSupported(); +} + +static Cache::CacheItemHelper kBasicCacheItemHelper(CacheEntryRole::kMisc, + &NoopDelete); +} // namespace + +const Cache::CacheItemHelper kSliceCacheItemHelper{ + CacheEntryRole::kMisc, &NoopDelete, &SliceSize, + &SliceSaveTo, &NoopCreate, &kBasicCacheItemHelper, +}; + Status SecondaryCache::CreateFromString( const ConfigOptions& config_options, const std::string& value, std::shared_ptr* result) { diff --git a/cache/cache_bench_tool.cc b/cache/cache_bench_tool.cc index 61b12b19e..07cd1a1f6 100644 --- a/cache/cache_bench_tool.cc +++ b/cache/cache_bench_tool.cc @@ -3,7 +3,6 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -#include "cache_key.h" #ifdef GFLAGS #include #include @@ -13,9 +12,12 @@ #include #include +#include "cache/cache_key.h" +#include "cache/sharded_cache.h" #include "db/db_impl/db_impl.h" #include "monitoring/histogram.h" #include "port/port.h" +#include "port/stack_trace.h" #include "rocksdb/advanced_cache.h" #include "rocksdb/convenience.h" #include "rocksdb/db.h" @@ -31,6 +33,7 @@ #include "util/hash.h" #include "util/mutexlock.h" #include "util/random.h" +#include "util/stderr_logger.h" #include "util/stop_watch.h" #include "util/string_util.h" @@ -43,21 +46,43 @@ static constexpr uint64_t GiB = MiB << 10; DEFINE_uint32(threads, 16, "Number of concurrent threads to run."); DEFINE_uint64(cache_size, 1 * GiB, "Number of bytes to use as a cache of uncompressed data."); -DEFINE_uint32(num_shard_bits, 6, "shard_bits."); +DEFINE_int32(num_shard_bits, -1, + "ShardedCacheOptions::shard_bits. Default = auto"); +DEFINE_int32( + eviction_effort_cap, + ROCKSDB_NAMESPACE::HyperClockCacheOptions(1, 1).eviction_effort_cap, + "HyperClockCacheOptions::eviction_effort_cap"); DEFINE_double(resident_ratio, 0.25, "Ratio of keys fitting in cache to keyspace."); DEFINE_uint64(ops_per_thread, 2000000U, "Number of operations per thread."); DEFINE_uint32(value_bytes, 8 * KiB, "Size of each value added."); - -DEFINE_uint32(skew, 5, "Degree of skew in key selection"); +DEFINE_uint32(value_bytes_estimate, 0, + "If > 0, overrides estimated_entry_charge or " + "min_avg_entry_charge depending on cache_type."); + +DEFINE_int32( + degenerate_hash_bits, 0, + "With HCC, fix this many hash bits to increase table hash collisions"); +DEFINE_uint32(skew, 5, "Degree of skew in key selection. 0 = no skew"); DEFINE_bool(populate_cache, true, "Populate cache before operations"); -DEFINE_uint32(lookup_insert_percent, 87, +DEFINE_double(pinned_ratio, 0.25, + "Keep roughly this portion of entries pinned in cache."); +DEFINE_double( + vary_capacity_ratio, 0.0, + "If greater than 0.0, will periodically vary the capacity between this " + "ratio less than full size and full size. If vary_capacity_ratio + " + "pinned_ratio is close to or exceeds 1.0, the cache might thrash."); + +DEFINE_uint32(lookup_insert_percent, 82, "Ratio of lookup (+ insert on not found) to total workload " "(expressed as a percentage)"); DEFINE_uint32(insert_percent, 2, "Ratio of insert to total workload (expressed as a percentage)"); +DEFINE_uint32(blind_insert_percent, 5, + "Ratio of insert without keeping handle to total workload " + "(expressed as a percentage)"); DEFINE_uint32(lookup_percent, 10, "Ratio of lookup to total workload (expressed as a percentage)"); DEFINE_uint32(erase_percent, 1, @@ -71,7 +96,8 @@ DEFINE_uint32( DEFINE_uint32(gather_stats_entries_per_lock, 256, "For Cache::ApplyToAllEntries"); -DEFINE_bool(skewed, false, "If true, skew the key access distribution"); + +DEFINE_uint32(usleep, 0, "Sleep up to this many microseconds after each op."); DEFINE_bool(lean, false, "If true, no additional computation is performed besides cache " @@ -81,12 +107,29 @@ DEFINE_bool(early_exit, false, "Exit before deallocating most memory. Good for malloc stats, e.g." "MALLOC_CONF=\"stats_print:true\""); +DEFINE_bool(histograms, true, + "Whether to track and print histogram statistics."); + +DEFINE_bool(report_problems, true, "Whether to ReportProblems() at the end."); + +DEFINE_uint32(seed, 0, "Hashing/random seed to use. 0 = choose at random"); + DEFINE_string(secondary_cache_uri, "", "Full URI for creating a custom secondary cache object"); -static class std::shared_ptr secondary_cache; DEFINE_string(cache_type, "lru_cache", "Type of block cache."); +DEFINE_bool(use_jemalloc_no_dump_allocator, false, + "Whether to use JemallocNoDumpAllocator"); + +DEFINE_uint32(jemalloc_no_dump_allocator_num_arenas, + ROCKSDB_NAMESPACE::JemallocAllocatorOptions().num_arenas, + "JemallocNodumpAllocator::num_arenas"); + +DEFINE_bool(jemalloc_no_dump_allocator_limit_tcache_size, + ROCKSDB_NAMESPACE::JemallocAllocatorOptions().limit_tcache_size, + "JemallocNodumpAllocator::limit_tcache_size"); + // ## BEGIN stress_cache_key sub-tool options ## // See class StressCacheKey below. DEFINE_bool(stress_cache_key, false, @@ -149,9 +192,6 @@ class SharedState { public: explicit SharedState(CacheBench* cache_bench) : cv_(&mu_), - num_initialized_(0), - start_(false), - num_done_(0), cache_bench_(cache_bench) {} ~SharedState() {} @@ -174,15 +214,31 @@ class SharedState { bool Started() const { return start_; } + void AddLookupStats(uint64_t hits, uint64_t misses, size_t pinned_count) { + MutexLock l(&mu_); + lookup_count_ += hits + misses; + lookup_hits_ += hits; + pinned_count_ += pinned_count; + } + + double GetLookupHitRatio() const { + return 1.0 * lookup_hits_ / lookup_count_; + } + + size_t GetPinnedCount() const { return pinned_count_; } + private: port::Mutex mu_; port::CondVar cv_; - uint64_t num_initialized_; - bool start_; - uint64_t num_done_; - CacheBench* cache_bench_; + + uint64_t num_initialized_ = 0; + bool start_ = false; + uint64_t num_done_ = 0; + uint64_t lookup_count_ = 0; + uint64_t lookup_hits_ = 0; + size_t pinned_count_ = 0; }; // Per-thread state for concurrent executions of the same benchmark. @@ -194,26 +250,32 @@ struct ThreadState { uint64_t duration_us = 0; ThreadState(uint32_t index, SharedState* _shared) - : tid(index), rnd(1000 + index), shared(_shared) {} + : tid(index), rnd(FLAGS_seed + 1 + index), shared(_shared) {} }; struct KeyGen { char key_data[27]; - Slice GetRand(Random64& rnd, uint64_t max_key, int max_log) { - uint64_t key = 0; - if (!FLAGS_skewed) { - uint64_t raw = rnd.Next(); - // Skew according to setting - for (uint32_t i = 0; i < FLAGS_skew; ++i) { - raw = std::min(raw, rnd.Next()); - } - key = FastRange64(raw, max_key); - } else { - key = rnd.Skewed(max_log); - if (key > max_key) { - key -= max_key; - } + Slice GetRand(Random64& rnd, uint64_t max_key, uint32_t skew) { + uint64_t raw = rnd.Next(); + // Skew according to setting + for (uint32_t i = 0; i < skew; ++i) { + raw = std::min(raw, rnd.Next()); + } + uint64_t key = FastRange64(raw, max_key); + if (FLAGS_degenerate_hash_bits) { + uint64_t key_hash = + Hash64(reinterpret_cast(&key), sizeof(key)); + // HCC uses the high 64 bits and a lower bit mask for starting probe + // location, so we fix hash bits starting at the bottom of that word. + auto hi_hash = uint64_t{0x9e3779b97f4a7c13U} ^ + (key_hash << 1 << (FLAGS_degenerate_hash_bits - 1)); + uint64_t un_hi, un_lo; + BijectiveUnhash2x64(hi_hash, key_hash, &un_hi, &un_lo); + un_lo ^= BitwiseAnd(FLAGS_seed, INT32_MAX); + EncodeFixed64(key_data, un_lo); + EncodeFixed64(key_data + 8, un_hi); + return Slice(key_data, kCacheKeySize); } // Variable size and alignment size_t off = key % 8; @@ -228,8 +290,8 @@ struct KeyGen { } }; -Cache::ObjectPtr createValue(Random64& rnd) { - char* rv = new char[FLAGS_value_bytes]; +Cache::ObjectPtr createValue(Random64& rnd, MemoryAllocator* alloc) { + char* rv = AllocateBlock(FLAGS_value_bytes, alloc).release(); // Fill with some filler data, and take some CPU time for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) { EncodeFixed64(rv + i, rnd.Next()); @@ -246,7 +308,8 @@ Status SaveToFn(Cache::ObjectPtr from_obj, size_t /*from_offset*/, return Status::OK(); } -Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/, +Status CreateFn(const Slice& data, CompressionType /*type*/, + CacheTier /*source*/, Cache::CreateContext* /*context*/, MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj, size_t* out_charge) { *out_obj = new char[data.size()]; @@ -255,8 +318,8 @@ Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/, return Status::OK(); }; -void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { - delete[] static_cast(value); +void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* alloc) { + CustomDeleter{alloc}(static_cast(value)); } Cache::CacheItemHelper helper1_wos(CacheEntryRole::kDataBlock, DeleteFn); @@ -268,6 +331,28 @@ Cache::CacheItemHelper helper2(CacheEntryRole::kIndexBlock, DeleteFn, SizeFn, Cache::CacheItemHelper helper3_wos(CacheEntryRole::kFilterBlock, DeleteFn); Cache::CacheItemHelper helper3(CacheEntryRole::kFilterBlock, DeleteFn, SizeFn, SaveToFn, CreateFn, &helper3_wos); + +void ConfigureSecondaryCache(ShardedCacheOptions& opts) { + if (!FLAGS_secondary_cache_uri.empty()) { + std::shared_ptr secondary_cache; + Status s = SecondaryCache::CreateFromString( + ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache); + if (secondary_cache == nullptr) { + fprintf(stderr, + "No secondary cache registered matching string: %s status=%s\n", + FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str()); + exit(1); + } + opts.secondary_cache = secondary_cache; + } +} + +ShardedCacheBase* AsShardedCache(Cache* c) { + if (!FLAGS_secondary_cache_uri.empty()) { + c = static_cast_with_check(c)->GetTarget().get(); + } + return static_cast_with_check(c); +} } // namespace class CacheBench { @@ -282,50 +367,60 @@ class CacheBench { FLAGS_lookup_insert_percent), insert_threshold_(lookup_insert_threshold_ + kHundredthUint64 * FLAGS_insert_percent), - lookup_threshold_(insert_threshold_ + + blind_insert_threshold_(insert_threshold_ + + kHundredthUint64 * FLAGS_blind_insert_percent), + lookup_threshold_(blind_insert_threshold_ + kHundredthUint64 * FLAGS_lookup_percent), erase_threshold_(lookup_threshold_ + - kHundredthUint64 * FLAGS_erase_percent), - skewed_(FLAGS_skewed) { + kHundredthUint64 * FLAGS_erase_percent) { if (erase_threshold_ != 100U * kHundredthUint64) { fprintf(stderr, "Percentages must add to 100.\n"); exit(1); } - max_log_ = 0; - if (skewed_) { - uint64_t max_key = max_key_; - while (max_key >>= 1) max_log_++; - if (max_key > (static_cast(1) << max_log_)) max_log_++; + std::shared_ptr allocator; + if (FLAGS_use_jemalloc_no_dump_allocator) { + JemallocAllocatorOptions opts; + opts.num_arenas = FLAGS_jemalloc_no_dump_allocator_num_arenas; + opts.limit_tcache_size = + FLAGS_jemalloc_no_dump_allocator_limit_tcache_size; + Status s = NewJemallocNodumpAllocator(opts, &allocator); + assert(s.ok()); } - if (FLAGS_cache_type == "clock_cache") { fprintf(stderr, "Old clock cache implementation has been removed.\n"); exit(1); - } else if (FLAGS_cache_type == "hyper_clock_cache") { - cache_ = HyperClockCacheOptions(FLAGS_cache_size, FLAGS_value_bytes, - FLAGS_num_shard_bits) - .MakeSharedCache(); + } else if (EndsWith(FLAGS_cache_type, "hyper_clock_cache")) { + HyperClockCacheOptions opts( + FLAGS_cache_size, /*estimated_entry_charge=*/0, FLAGS_num_shard_bits); + opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX); + opts.memory_allocator = allocator; + opts.eviction_effort_cap = FLAGS_eviction_effort_cap; + if (FLAGS_cache_type == "fixed_hyper_clock_cache" || + FLAGS_cache_type == "hyper_clock_cache") { + opts.estimated_entry_charge = FLAGS_value_bytes_estimate > 0 + ? FLAGS_value_bytes_estimate + : FLAGS_value_bytes; + } else if (FLAGS_cache_type == "auto_hyper_clock_cache") { + if (FLAGS_value_bytes_estimate > 0) { + opts.min_avg_entry_charge = FLAGS_value_bytes_estimate; + } + } else { + fprintf(stderr, "Cache type not supported.\n"); + exit(1); + } + ConfigureSecondaryCache(opts); + cache_ = opts.MakeSharedCache(); } else if (FLAGS_cache_type == "lru_cache") { LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */); - if (!FLAGS_secondary_cache_uri.empty()) { - Status s = SecondaryCache::CreateFromString( - ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache); - if (secondary_cache == nullptr) { - fprintf( - stderr, - "No secondary cache registered matching string: %s status=%s\n", - FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str()); - exit(1); - } - opts.secondary_cache = secondary_cache; - } - + opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX); + opts.memory_allocator = allocator; + ConfigureSecondaryCache(opts); cache_ = NewLRUCache(opts); } else { - fprintf(stderr, "Cache type not supported."); + fprintf(stderr, "Cache type not supported.\n"); exit(1); } } @@ -333,13 +428,51 @@ class CacheBench { ~CacheBench() {} void PopulateCache() { - Random64 rnd(1); + Random64 rnd(FLAGS_seed); KeyGen keygen; - for (uint64_t i = 0; i < 2 * FLAGS_cache_size; i += FLAGS_value_bytes) { - Status s = cache_->Insert(keygen.GetRand(rnd, max_key_, max_log_), - createValue(rnd), &helper1, FLAGS_value_bytes); + size_t max_occ = 0; + size_t inserts_since_max_occ_increase = 0; + size_t keys_since_last_not_found = 0; + + // Avoid redundant insertions by checking Lookup before Insert. + // Loop until insertions consistently fail to increase max occupancy or + // it becomes difficult to find keys not already inserted. + while (inserts_since_max_occ_increase < 100 && + keys_since_last_not_found < 100) { + Slice key = keygen.GetRand(rnd, max_key_, FLAGS_skew); + + Cache::Handle* handle = cache_->Lookup(key); + if (handle != nullptr) { + cache_->Release(handle); + ++keys_since_last_not_found; + continue; + } + keys_since_last_not_found = 0; + + Status s = + cache_->Insert(key, createValue(rnd, cache_->memory_allocator()), + &helper1, FLAGS_value_bytes); assert(s.ok()); + + handle = cache_->Lookup(key); + if (!handle) { + fprintf(stderr, "Failed to lookup key just inserted.\n"); + assert(false); + exit(42); + } else { + cache_->Release(handle); + } + + size_t occ = cache_->GetOccupancyCount(); + if (occ > max_occ) { + max_occ = occ; + inserts_since_max_occ_increase = 0; + } else { + ++inserts_since_max_occ_increase; + } } + printf("Population complete (%zu entries, %g average charge)\n", max_occ, + 1.0 * FLAGS_cache_size / max_occ); } bool Run() { @@ -398,19 +531,35 @@ class CacheBench { FLAGS_ops_per_thread / elapsed_secs); printf("Thread ops/sec = %u\n", ops_per_sec); - printf("\nOperation latency (ns):\n"); - HistogramImpl combined; - for (uint32_t i = 0; i < FLAGS_threads; i++) { - combined.Merge(threads[i]->latency_ns_hist); - } - printf("%s", combined.ToString().c_str()); + printf("Lookup hit ratio: %g\n", shared.GetLookupHitRatio()); - if (FLAGS_gather_stats) { - printf("\nGather stats latency (us):\n"); - printf("%s", stats_hist.ToString().c_str()); + size_t occ = cache_->GetOccupancyCount(); + size_t slot = cache_->GetTableAddressCount(); + printf("Final load factor: %g (%zu / %zu)\n", 1.0 * occ / slot, occ, slot); + + printf("Final pinned count: %zu\n", shared.GetPinnedCount()); + + if (FLAGS_histograms) { + printf("\nOperation latency (ns):\n"); + HistogramImpl combined; + for (uint32_t i = 0; i < FLAGS_threads; i++) { + combined.Merge(threads[i]->latency_ns_hist); + } + printf("%s", combined.ToString().c_str()); + + if (FLAGS_gather_stats) { + printf("\nGather stats latency (us):\n"); + printf("%s", stats_hist.ToString().c_str()); + } } - printf("\n%s", stats_report.c_str()); + if (FLAGS_report_problems) { + printf("\n"); + std::shared_ptr logger = + std::make_shared(InfoLogLevel::DEBUG_LEVEL); + cache_->ReportProblems(logger); + } + printf("%s", stats_report.c_str()); return true; } @@ -421,10 +570,9 @@ class CacheBench { // Cumulative thresholds in the space of a random uint64_t const uint64_t lookup_insert_threshold_; const uint64_t insert_threshold_; + const uint64_t blind_insert_threshold_; const uint64_t lookup_threshold_; const uint64_t erase_threshold_; - const bool skewed_; - int max_log_; // A benchmark version of gathering stats on an active block cache by // iterating over it. The primary purpose is to measure the impact of @@ -457,7 +605,7 @@ class CacheBench { for (;;) { if (shared->AllDone()) { std::ostringstream ostr; - ostr << "Most recent cache entry stats:\n" + ostr << "\nMost recent cache entry stats:\n" << "Number of entries: " << total_entry_count << "\n" << "Table occupancy: " << table_occupancy << " / " << table_size << " = " @@ -494,13 +642,17 @@ class CacheBench { // Something slightly more expensive as in stats by category helpers.insert(helper); }; - timer.Start(); + if (FLAGS_histograms) { + timer.Start(); + } Cache::ApplyToAllEntriesOptions opts; opts.average_entries_per_lock = FLAGS_gather_stats_entries_per_lock; shared->GetCacheBench()->cache_->ApplyToAllEntries(fn, opts); table_occupancy = shared->GetCacheBench()->cache_->GetOccupancyCount(); table_size = shared->GetCacheBench()->cache_->GetTableAddressCount(); - stats_hist->Add(timer.ElapsedNanos() / 1000); + if (FLAGS_histograms) { + stats_hist->Add(timer.ElapsedNanos() / 1000); + } } } @@ -531,62 +683,89 @@ class CacheBench { void OperateCache(ThreadState* thread) { // To use looked-up values uint64_t result = 0; + uint64_t lookup_misses = 0; + uint64_t lookup_hits = 0; // To hold handles for a non-trivial amount of time - Cache::Handle* handle = nullptr; + std::deque pinned; + size_t total_pin_count = static_cast( + (FLAGS_cache_size * FLAGS_pinned_ratio) / FLAGS_value_bytes + 0.999999); + // For this thread. Some round up, some round down, as appropriate + size_t pin_count = (total_pin_count + thread->tid) / FLAGS_threads; + KeyGen gen; const auto clock = SystemClock::Default().get(); uint64_t start_time = clock->NowMicros(); StopWatchNano timer(clock); + auto system_clock = SystemClock::Default(); + size_t steps_to_next_capacity_change = 0; for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) { - Slice key = gen.GetRand(thread->rnd, max_key_, max_log_); + Slice key = gen.GetRand(thread->rnd, max_key_, FLAGS_skew); uint64_t random_op = thread->rnd.Next(); - timer.Start(); + if (FLAGS_vary_capacity_ratio > 0.0 && thread->tid == 0) { + if (steps_to_next_capacity_change == 0) { + double cut_ratio = static_cast(thread->rnd.Next()) / + static_cast(UINT64_MAX) * + FLAGS_vary_capacity_ratio; + cache_->SetCapacity(FLAGS_cache_size * (1.0 - cut_ratio)); + steps_to_next_capacity_change = + static_cast(FLAGS_ops_per_thread / 100); + } else { + --steps_to_next_capacity_change; + } + } + + if (FLAGS_histograms) { + timer.Start(); + } if (random_op < lookup_insert_threshold_) { - if (handle) { - cache_->Release(handle); - handle = nullptr; - } // do lookup - handle = cache_->Lookup(key, &helper2, /*context*/ nullptr, - Cache::Priority::LOW); + auto handle = cache_->Lookup(key, &helper2, /*context*/ nullptr, + Cache::Priority::LOW); if (handle) { + ++lookup_hits; if (!FLAGS_lean) { // do something with the data result += NPHash64(static_cast(cache_->Value(handle)), FLAGS_value_bytes); } + pinned.push_back(handle); } else { + ++lookup_misses; // do insert - Status s = cache_->Insert(key, createValue(thread->rnd), &helper2, - FLAGS_value_bytes, &handle); + Status s = cache_->Insert( + key, createValue(thread->rnd, cache_->memory_allocator()), + &helper2, FLAGS_value_bytes, &pinned.emplace_back()); assert(s.ok()); } } else if (random_op < insert_threshold_) { - if (handle) { - cache_->Release(handle); - handle = nullptr; - } // do insert - Status s = cache_->Insert(key, createValue(thread->rnd), &helper3, - FLAGS_value_bytes, &handle); + Status s = cache_->Insert( + key, createValue(thread->rnd, cache_->memory_allocator()), &helper3, + FLAGS_value_bytes, &pinned.emplace_back()); + assert(s.ok()); + } else if (random_op < blind_insert_threshold_) { + // insert without keeping a handle + Status s = cache_->Insert( + key, createValue(thread->rnd, cache_->memory_allocator()), &helper3, + FLAGS_value_bytes); assert(s.ok()); } else if (random_op < lookup_threshold_) { - if (handle) { - cache_->Release(handle); - handle = nullptr; - } // do lookup - handle = cache_->Lookup(key, &helper2, /*context*/ nullptr, - Cache::Priority::LOW); + auto handle = cache_->Lookup(key, &helper2, /*context*/ nullptr, + Cache::Priority::LOW); if (handle) { + ++lookup_hits; if (!FLAGS_lean) { // do something with the data result += NPHash64(static_cast(cache_->Value(handle)), FLAGS_value_bytes); } + pinned.push_back(handle); + } else { + ++lookup_misses; } } else if (random_op < erase_threshold_) { // do erase @@ -595,13 +774,27 @@ class CacheBench { // Should be extremely unlikely (noop) assert(random_op >= kHundredthUint64 * 100U); } - thread->latency_ns_hist.Add(timer.ElapsedNanos()); + if (FLAGS_histograms) { + thread->latency_ns_hist.Add(timer.ElapsedNanos()); + } + if (FLAGS_usleep > 0) { + unsigned us = + static_cast(thread->rnd.Uniform(FLAGS_usleep + 1)); + if (us > 0) { + system_clock->SleepForMicroseconds(us); + } + } + while (pinned.size() > pin_count) { + cache_->Release(pinned.front()); + pinned.pop_front(); + } } if (FLAGS_early_exit) { MutexLock l(thread->shared->GetMutex()); exit(0); } - if (handle) { + thread->shared->AddLookupStats(lookup_hits, lookup_misses, pinned.size()); + for (auto handle : pinned) { cache_->Release(handle); handle = nullptr; } @@ -621,13 +814,16 @@ class CacheBench { #ifndef NDEBUG printf("WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); #endif + printf("----------------------------\n"); printf("RocksDB version : %d.%d\n", kMajorVersion, kMinorVersion); + printf("Cache impl name : %s\n", cache_->Name()); printf("DMutex impl name : %s\n", DMutex::kName()); printf("Number of threads : %u\n", FLAGS_threads); printf("Ops per thread : %" PRIu64 "\n", FLAGS_ops_per_thread); printf("Cache size : %s\n", BytesToHumanString(FLAGS_cache_size).c_str()); - printf("Num shard bits : %u\n", FLAGS_num_shard_bits); + printf("Num shard bits : %d\n", + AsShardedCache(cache_.get())->GetNumShardBits()); printf("Max key : %" PRIu64 "\n", max_key_); printf("Resident ratio : %g\n", FLAGS_resident_ratio); printf("Skew degree : %u\n", FLAGS_skew); @@ -947,6 +1143,7 @@ class StressCacheKey { }; int cache_bench_tool(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); ParseCommandLineFlags(&argc, &argv, true); if (FLAGS_stress_cache_key) { @@ -960,11 +1157,14 @@ int cache_bench_tool(int argc, char** argv) { exit(1); } + if (FLAGS_seed == 0) { + FLAGS_seed = static_cast(port::GetProcessID()); + printf("Using seed = %" PRIu32 "\n", FLAGS_seed); + } + ROCKSDB_NAMESPACE::CacheBench bench; if (FLAGS_populate_cache) { bench.PopulateCache(); - printf("Population complete\n"); - printf("----------------------------\n"); } if (bench.Run()) { return 0; diff --git a/cache/cache_helpers.cc b/cache/cache_helpers.cc index 22597bf6d..bceb6f3c0 100644 --- a/cache/cache_helpers.cc +++ b/cache/cache_helpers.cc @@ -25,7 +25,8 @@ Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved, assert(helper->create_cb); Cache::ObjectPtr value; size_t charge; - Status st = helper->create_cb(saved, create_context, + Status st = helper->create_cb(saved, CompressionType::kNoCompression, + CacheTier::kVolatileTier, create_context, cache->memory_allocator(), &value, &charge); if (st.ok()) { st = diff --git a/cache/cache_reservation_manager.h b/cache/cache_reservation_manager.h index 08bf59b00..a7b06dea2 100644 --- a/cache/cache_reservation_manager.h +++ b/cache/cache_reservation_manager.h @@ -273,9 +273,10 @@ class ConcurrentCacheReservationManager std::size_t total_mem_used = cache_res_mgr_->GetTotalMemoryUsed(); Status s; if (!increase) { - assert(total_mem_used >= memory_used_delta); - s = cache_res_mgr_->UpdateCacheReservation(total_mem_used - - memory_used_delta); + s = cache_res_mgr_->UpdateCacheReservation( + (total_mem_used > memory_used_delta) + ? (total_mem_used - memory_used_delta) + : 0); } else { s = cache_res_mgr_->UpdateCacheReservation(total_mem_used + memory_used_delta); diff --git a/cache/cache_test.cc b/cache/cache_test.cc index 4585faacb..f21efc47a 100644 --- a/cache/cache_test.cc +++ b/cache/cache_test.cc @@ -70,18 +70,11 @@ const Cache::CacheItemHelper kDumbHelper{ CacheEntryRole::kMisc, [](Cache::ObjectPtr /*value*/, MemoryAllocator* /*alloc*/) {}}; -const Cache::CacheItemHelper kEraseOnDeleteHelper1{ +const Cache::CacheItemHelper kInvokeOnDeleteHelper{ CacheEntryRole::kMisc, [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { - Cache* cache = static_cast(value); - cache->Erase("foo"); - }}; - -const Cache::CacheItemHelper kEraseOnDeleteHelper2{ - CacheEntryRole::kMisc, - [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { - Cache* cache = static_cast(value); - cache->Erase(EncodeKey16Bytes(1234)); + auto& fn = *static_cast*>(value); + fn(); }}; } // anonymous namespace @@ -120,8 +113,7 @@ class CacheTest : public testing::Test, // Currently, HyperClockCache requires keys to be 16B long, whereas // LRUCache doesn't, so the encoding depends on the cache type. std::string EncodeKey(int k) { - auto type = GetParam(); - if (type == kHyperClock) { + if (IsHyperClock()) { return EncodeKey16Bytes(k); } else { return EncodeKey32Bits(k); @@ -129,8 +121,7 @@ class CacheTest : public testing::Test, } int DecodeKey(const Slice& k) { - auto type = GetParam(); - if (type == kHyperClock) { + if (IsHyperClock()) { return DecodeKey16Bytes(k); } else { return DecodeKey32Bits(k); @@ -182,15 +173,13 @@ std::string CacheTest::type_; class LRUCacheTest : public CacheTest {}; TEST_P(CacheTest, UsageTest) { - auto type = GetParam(); - // cache is std::shared_ptr and will be automatically cleaned up. const size_t kCapacity = 100000; - auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata); + auto cache = NewCache(kCapacity, 6, false, kDontChargeCacheMetadata); auto precise_cache = NewCache(kCapacity, 0, false, kFullChargeCacheMetadata); ASSERT_EQ(0, cache->GetUsage()); size_t baseline_meta_usage = precise_cache->GetUsage(); - if (type != kHyperClock) { + if (!IsHyperClock()) { ASSERT_EQ(0, baseline_meta_usage); } @@ -198,20 +187,19 @@ TEST_P(CacheTest, UsageTest) { char value[10] = "abcdef"; // make sure everything will be cached for (int i = 1; i < 100; ++i) { - std::string key; - if (type == kLRU) { - key = std::string(i, 'a'); - } else { - key = EncodeKey(i); - } + std::string key = EncodeKey(i); auto kv_size = key.size() + 5; ASSERT_OK(cache->Insert(key, value, &kDumbHelper, kv_size)); ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size)); usage += kv_size; ASSERT_EQ(usage, cache->GetUsage()); - if (type == kHyperClock) { + if (GetParam() == kFixedHyperClock) { ASSERT_EQ(baseline_meta_usage + usage, precise_cache->GetUsage()); } else { + // AutoHyperClockCache meta usage grows in proportion to lifetime + // max number of entries. LRUCache in proportion to resident number of + // entries, though there is an untracked component proportional to + // lifetime max number of entries. ASSERT_LT(usage, precise_cache->GetUsage()); } } @@ -219,16 +207,15 @@ TEST_P(CacheTest, UsageTest) { cache->EraseUnRefEntries(); precise_cache->EraseUnRefEntries(); ASSERT_EQ(0, cache->GetUsage()); - ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage()); + if (GetParam() != kAutoHyperClock) { + // NOTE: AutoHyperClockCache meta usage grows in proportion to lifetime + // max number of entries. + ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage()); + } // make sure the cache will be overloaded for (size_t i = 1; i < kCapacity; ++i) { - std::string key; - if (type == kLRU) { - key = std::to_string(i); - } else { - key = EncodeKey(static_cast(1000 + i)); - } + std::string key = EncodeKey(static_cast(1000 + i)); ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5)); ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5)); } @@ -237,7 +224,7 @@ TEST_P(CacheTest, UsageTest) { ASSERT_GT(kCapacity, cache->GetUsage()); ASSERT_GT(kCapacity, precise_cache->GetUsage()); ASSERT_LT(kCapacity * 0.95, cache->GetUsage()); - if (type != kHyperClock) { + if (!IsHyperClock()) { ASSERT_LT(kCapacity * 0.95, precise_cache->GetUsage()); } else { // estimated value size of 1 is weird for clock cache, because @@ -248,22 +235,20 @@ TEST_P(CacheTest, UsageTest) { } } -// TODO: This test takes longer than expected on ClockCache. This is -// because the values size estimate at construction is too sloppy. +// TODO: This test takes longer than expected on FixedHyperClockCache. +// This is because the values size estimate at construction is too sloppy. // Fix this. // Why is it so slow? The cache is constructed with an estimate of 1, but // then the charge is claimed to be 21. This will cause the hash table // to be extremely sparse, which in turn means clock needs to scan too // many slots to find victims. TEST_P(CacheTest, PinnedUsageTest) { - auto type = GetParam(); - // cache is std::shared_ptr and will be automatically cleaned up. const size_t kCapacity = 200000; auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata); auto precise_cache = NewCache(kCapacity, 8, false, kFullChargeCacheMetadata); size_t baseline_meta_usage = precise_cache->GetUsage(); - if (type != kHyperClock) { + if (!IsHyperClock()) { ASSERT_EQ(0, baseline_meta_usage); } @@ -276,12 +261,7 @@ TEST_P(CacheTest, PinnedUsageTest) { // Add entries. Unpin some of them after insertion. Then, pin some of them // again. Check GetPinnedUsage(). for (int i = 1; i < 100; ++i) { - std::string key; - if (type == kLRU) { - key = std::string(i, 'a'); - } else { - key = EncodeKey(i); - } + std::string key = EncodeKey(i); auto kv_size = key.size() + 5; Cache::Handle* handle; Cache::Handle* handle_in_precise_cache; @@ -322,12 +302,7 @@ TEST_P(CacheTest, PinnedUsageTest) { // check that overloading the cache does not change the pinned usage for (size_t i = 1; i < 2 * kCapacity; ++i) { - std::string key; - if (type == kLRU) { - key = std::to_string(i); - } else { - key = EncodeKey(static_cast(1000 + i)); - } + std::string key = EncodeKey(static_cast(1000 + i)); ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5)); ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5)); } @@ -351,7 +326,11 @@ TEST_P(CacheTest, PinnedUsageTest) { cache->EraseUnRefEntries(); precise_cache->EraseUnRefEntries(); ASSERT_EQ(0, cache->GetUsage()); - ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage()); + if (GetParam() != kAutoHyperClock) { + // NOTE: AutoHyperClockCache meta usage grows in proportion to lifetime + // max number of entries. + ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage()); + } } TEST_P(CacheTest, HitAndMiss) { @@ -368,7 +347,7 @@ TEST_P(CacheTest, HitAndMiss) { ASSERT_EQ(-1, Lookup(300)); Insert(100, 102); - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { // ClockCache usually doesn't overwrite on Insert ASSERT_EQ(101, Lookup(100)); } else { @@ -378,7 +357,7 @@ TEST_P(CacheTest, HitAndMiss) { ASSERT_EQ(-1, Lookup(300)); ASSERT_EQ(1U, deleted_values_.size()); - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { ASSERT_EQ(102, deleted_values_[0]); } else { ASSERT_EQ(101, deleted_values_[0]); @@ -386,7 +365,7 @@ TEST_P(CacheTest, HitAndMiss) { } TEST_P(CacheTest, InsertSameKey) { - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { ROCKSDB_GTEST_BYPASS( "ClockCache doesn't guarantee Insert overwrite same key."); return; @@ -415,7 +394,7 @@ TEST_P(CacheTest, Erase) { } TEST_P(CacheTest, EntriesArePinned) { - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { ROCKSDB_GTEST_BYPASS( "ClockCache doesn't guarantee Insert overwrite same key."); return; @@ -479,7 +458,7 @@ TEST_P(CacheTest, ExternalRefPinsEntries) { Insert(1000 + j, 2000 + j); } // Clock cache is even more stateful and needs more churn to evict - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { for (int j = 0; j < kCacheSize; j++) { Insert(11000 + j, 11000 + j); } @@ -517,20 +496,20 @@ TEST_P(CacheTest, EvictionPolicyRef) { // Check whether the entries inserted in the beginning // are evicted. Ones without extra ref are evicted and // those with are not. - ASSERT_EQ(-1, Lookup(100)); - ASSERT_EQ(-1, Lookup(101)); - ASSERT_EQ(-1, Lookup(102)); - ASSERT_EQ(-1, Lookup(103)); + EXPECT_EQ(-1, Lookup(100)); + EXPECT_EQ(-1, Lookup(101)); + EXPECT_EQ(-1, Lookup(102)); + EXPECT_EQ(-1, Lookup(103)); - ASSERT_EQ(-1, Lookup(300)); - ASSERT_EQ(-1, Lookup(301)); - ASSERT_EQ(-1, Lookup(302)); - ASSERT_EQ(-1, Lookup(303)); + EXPECT_EQ(-1, Lookup(300)); + EXPECT_EQ(-1, Lookup(301)); + EXPECT_EQ(-1, Lookup(302)); + EXPECT_EQ(-1, Lookup(303)); - ASSERT_EQ(101, Lookup(200)); - ASSERT_EQ(102, Lookup(201)); - ASSERT_EQ(103, Lookup(202)); - ASSERT_EQ(104, Lookup(203)); + EXPECT_EQ(101, Lookup(200)); + EXPECT_EQ(102, Lookup(201)); + EXPECT_EQ(103, Lookup(202)); + EXPECT_EQ(104, Lookup(203)); // Cleaning up all the handles cache_->Release(h201); @@ -540,37 +519,22 @@ TEST_P(CacheTest, EvictionPolicyRef) { } TEST_P(CacheTest, EvictEmptyCache) { - auto type = GetParam(); - // Insert item large than capacity to trigger eviction on empty cache. auto cache = NewCache(1, 0, false); - if (type == kLRU) { - ASSERT_OK(cache->Insert("foo", nullptr, &kDumbHelper, 10)); - } else { - ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, &kDumbHelper, 10)); - } + ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, &kDumbHelper, 10)); } TEST_P(CacheTest, EraseFromDeleter) { - auto type = GetParam(); - // Have deleter which will erase item from cache, which will re-enter // the cache at that point. std::shared_ptr cache = NewCache(10, 0, false); - std::string foo, bar; - const Cache::CacheItemHelper* erase_helper; - if (type == kLRU) { - foo = "foo"; - bar = "bar"; - erase_helper = &kEraseOnDeleteHelper1; - } else { - foo = EncodeKey(1234); - bar = EncodeKey(5678); - erase_helper = &kEraseOnDeleteHelper2; - } + std::string foo = EncodeKey(1234); + std::string bar = EncodeKey(5678); + + std::function erase_fn = [&]() { cache->Erase(foo); }; ASSERT_OK(cache->Insert(foo, nullptr, &kDumbHelper, 1)); - ASSERT_OK(cache->Insert(bar, cache.get(), erase_helper, 1)); + ASSERT_OK(cache->Insert(bar, &erase_fn, &kInvokeOnDeleteHelper, 1)); cache->Erase(bar); ASSERT_EQ(nullptr, cache->Lookup(foo)); @@ -678,10 +642,10 @@ using TypedHandle = SharedCache::TypedHandle; } // namespace TEST_P(CacheTest, SetCapacity) { - auto type = GetParam(); - if (type == kHyperClock) { + if (IsHyperClock()) { + // TODO: update test & code for limited supoort ROCKSDB_GTEST_BYPASS( - "FastLRUCache and HyperClockCache don't support arbitrary capacity " + "HyperClockCache doesn't support arbitrary capacity " "adjustments."); return; } @@ -811,7 +775,7 @@ TEST_P(CacheTest, OverCapacity) { cache.Release(handles[i]); } - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { // Make sure eviction is triggered. ASSERT_OK(cache.Insert(EncodeKey(-1), nullptr, 1, &handles[0])); @@ -923,8 +887,7 @@ TEST_P(CacheTest, DefaultShardBits) { // Prevent excessive allocation (to save time & space) estimated_value_size_ = 100000; // Implementations use different minimum shard sizes - size_t min_shard_size = - (GetParam() == kHyperClock ? 32U * 1024U : 512U) * 1024U; + size_t min_shard_size = (IsHyperClock() ? 32U * 1024U : 512U) * 1024U; std::shared_ptr cache = NewCache(32U * min_shard_size); ShardedCacheBase* sc = dynamic_cast(cache.get()); diff --git a/cache/charged_cache.cc b/cache/charged_cache.cc index e44288ecd..6a21bacfb 100644 --- a/cache/charged_cache.cc +++ b/cache/charged_cache.cc @@ -19,8 +19,10 @@ ChargedCache::ChargedCache(std::shared_ptr cache, Status ChargedCache::Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, size_t charge, - Handle** handle, Priority priority) { - Status s = target_->Insert(key, obj, helper, charge, handle, priority); + Handle** handle, Priority priority, + const Slice& compressed_val, CompressionType type) { + Status s = target_->Insert(key, obj, helper, charge, handle, priority, + compressed_val, type); if (s.ok()) { // Insert may cause the cache entry eviction if the cache is full. So we // directly call the reservation manager to update the total memory used diff --git a/cache/charged_cache.h b/cache/charged_cache.h index f2eacb9ed..a59c178ab 100644 --- a/cache/charged_cache.h +++ b/cache/charged_cache.h @@ -22,9 +22,11 @@ class ChargedCache : public CacheWrapper { ChargedCache(std::shared_ptr cache, std::shared_ptr block_cache); - Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, - size_t charge, Handle** handle = nullptr, - Priority priority = Priority::LOW) override; + Status Insert( + const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW, const Slice& compressed_val = Slice(), + CompressionType type = CompressionType::kNoCompression) override; Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper, CreateContext* create_context, diff --git a/cache/clock_cache.cc b/cache/clock_cache.cc index e5a650d86..de2e56186 100644 --- a/cache/clock_cache.cc +++ b/cache/clock_cache.cc @@ -9,9 +9,19 @@ #include "cache/clock_cache.h" +#include +#include +#include #include +#include +#include +#include +#include #include #include +#include +#include +#include #include "cache/cache_key.h" #include "cache/secondary_cache_adapter.h" @@ -42,24 +52,25 @@ inline uint64_t GetInitialCountdown(Cache::Priority priority) { switch (priority) { case Cache::Priority::HIGH: return ClockHandle::kHighCountdown; - default: - assert(false); - FALLTHROUGH_INTENDED; case Cache::Priority::LOW: return ClockHandle::kLowCountdown; case Cache::Priority::BOTTOM: return ClockHandle::kBottomCountdown; } + // Switch should have been exhaustive. + assert(false); + // For release build, fall back on something reasonable. + return ClockHandle::kLowCountdown; } inline void MarkEmpty(ClockHandle& h) { #ifndef NDEBUG // Mark slot as empty, with assertion - uint64_t meta = h.meta.exchange(0, std::memory_order_release); + uint64_t meta = h.meta.Exchange(0); assert(meta >> ClockHandle::kStateShift == ClockHandle::kStateConstruction); #else // Mark slot as empty - h.meta.store(0, std::memory_order_release); + h.meta.Store(0); #endif } @@ -73,21 +84,51 @@ inline void FreeDataMarkEmpty(ClockHandle& h, MemoryAllocator* allocator) { MarkEmpty(h); } -inline bool ClockUpdate(ClockHandle& h) { - uint64_t meta = h.meta.load(std::memory_order_relaxed); +// Called to undo the effect of referencing an entry for internal purposes, +// so it should not be marked as having been used. +inline void Unref(const ClockHandle& h, uint64_t count = 1) { + // Pretend we never took the reference + // WART: there's a tiny chance we release last ref to invisible + // entry here. If that happens, we let eviction take care of it. + uint64_t old_meta = h.meta.FetchSub(ClockHandle::kAcquireIncrement * count); + assert(GetRefcount(old_meta) != 0); + (void)old_meta; +} + +inline bool ClockUpdate(ClockHandle& h, BaseClockTable::EvictionData* data, + bool* purgeable = nullptr) { + uint64_t meta; + if (purgeable) { + assert(*purgeable == false); + // In AutoHCC, our eviction process follows the chain structure, so we + // should ensure that we see the latest state of each entry, at least for + // assertion checking. + meta = h.meta.Load(); + } else { + // In FixedHCC, our eviction process is a simple iteration without regard + // to probing order, displacements, etc., so it doesn't matter if we see + // somewhat stale data. + meta = h.meta.LoadRelaxed(); + } + if (((meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit) == + 0) { + // Only clock update Shareable entries + if (purgeable) { + *purgeable = true; + // AutoHCC only: make sure we only attempt to update non-empty slots + assert((meta >> ClockHandle::kStateShift) & + ClockHandle::kStateOccupiedBit); + } + return false; + } uint64_t acquire_count = (meta >> ClockHandle::kAcquireCounterShift) & ClockHandle::kCounterMask; uint64_t release_count = (meta >> ClockHandle::kReleaseCounterShift) & ClockHandle::kCounterMask; - // fprintf(stderr, "ClockUpdate @ %p: %lu %lu %u\n", &h, acquire_count, - // release_count, (unsigned)(meta >> ClockHandle::kStateShift)); if (acquire_count != release_count) { // Only clock update entries with no outstanding refs - return false; - } - if (!((meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit)) { - // Only clock update Shareable entries + data->seen_pinned_count++; return false; } if ((meta >> ClockHandle::kStateShift == ClockHandle::kStateVisible) && @@ -99,18 +140,20 @@ inline bool ClockUpdate(ClockHandle& h) { // not aggressively uint64_t new_meta = (uint64_t{ClockHandle::kStateVisible} << ClockHandle::kStateShift) | + (meta & ClockHandle::kHitBitMask) | (new_count << ClockHandle::kReleaseCounterShift) | (new_count << ClockHandle::kAcquireCounterShift); - h.meta.compare_exchange_strong(meta, new_meta, std::memory_order_relaxed); + h.meta.CasStrongRelaxed(meta, new_meta); return false; } // Otherwise, remove entry (either unreferenced invisible or // unreferenced and expired visible). - if (h.meta.compare_exchange_strong( - meta, - uint64_t{ClockHandle::kStateConstruction} << ClockHandle::kStateShift, - std::memory_order_acquire)) { + if (h.meta.CasStrong(meta, (uint64_t{ClockHandle::kStateConstruction} + << ClockHandle::kStateShift) | + (meta & ClockHandle::kHitBitMask))) { // Took ownership. + data->freed_charge += h.GetTotalCharge(); + data->freed_count += 1; return true; } else { // Compare-exchange failing probably @@ -174,7 +217,7 @@ inline bool ClockUpdate(ClockHandle& h) { // motivates only checking for release counter in high state, not both in high // state.) inline void CorrectNearOverflow(uint64_t old_meta, - std::atomic& meta) { + AcqRelAtomic& meta) { // We clear both top-most counter bits at the same time. constexpr uint64_t kCounterTopBit = uint64_t{1} << (ClockHandle::kCounterNumBits - 1); @@ -188,7 +231,7 @@ inline void CorrectNearOverflow(uint64_t old_meta, << ClockHandle::kReleaseCounterShift; if (UNLIKELY(old_meta & kCheckBits)) { - meta.fetch_and(~kClearBits, std::memory_order_relaxed); + meta.FetchAndRelaxed(~kClearBits); } } @@ -197,9 +240,8 @@ inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h, assert(*already_matches == false); // Optimistically transition the slot from "empty" to // "under construction" (no effect on other states) - uint64_t old_meta = h.meta.fetch_or( - uint64_t{ClockHandle::kStateOccupiedBit} << ClockHandle::kStateShift, - std::memory_order_acq_rel); + uint64_t old_meta = h.meta.FetchOr(uint64_t{ClockHandle::kStateOccupiedBit} + << ClockHandle::kStateShift); uint64_t old_state = old_meta >> ClockHandle::kStateShift; if (old_state == ClockHandle::kStateEmpty) { @@ -215,41 +257,32 @@ inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h, // refs for initial countdown, so that we boost the clock state if // this is a match. old_meta = - h.meta.fetch_add(ClockHandle::kAcquireIncrement * initial_countdown, - std::memory_order_acq_rel); + h.meta.FetchAdd(ClockHandle::kAcquireIncrement * initial_countdown); // Like Lookup if ((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateVisible) { // Acquired a read reference if (h.hashed_key == proto.hashed_key) { // Match. Release in a way that boosts the clock state old_meta = - h.meta.fetch_add(ClockHandle::kReleaseIncrement * initial_countdown, - std::memory_order_acq_rel); + h.meta.FetchAdd(ClockHandle::kReleaseIncrement * initial_countdown); // Correct for possible (but rare) overflow CorrectNearOverflow(old_meta, h.meta); // Insert detached instead (only if return handle needed) *already_matches = true; return false; } else { - // Mismatch. Pretend we never took the reference - old_meta = - h.meta.fetch_sub(ClockHandle::kAcquireIncrement * initial_countdown, - std::memory_order_acq_rel); + // Mismatch. + Unref(h, initial_countdown); } } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateInvisible)) { // Pretend we never took the reference - // WART/FIXME?: there's a tiny chance we release last ref to invisible - // entry here. If that happens, we let eviction take care of it. - old_meta = - h.meta.fetch_sub(ClockHandle::kAcquireIncrement * initial_countdown, - std::memory_order_acq_rel); + Unref(h, initial_countdown); } else { // For other states, incrementing the acquire counter has no effect // so we don't need to undo it. // Slot not usable / touchable now. } - (void)old_meta; return false; } @@ -270,12 +303,12 @@ inline void FinishSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h, #ifndef NDEBUG // Save the state transition, with assertion - uint64_t old_meta = h.meta.exchange(new_meta, std::memory_order_release); + uint64_t old_meta = h.meta.Exchange(new_meta); assert(old_meta >> ClockHandle::kStateShift == ClockHandle::kStateConstruction); #else // Save the state transition - h.meta.store(new_meta, std::memory_order_release); + h.meta.Store(new_meta); #endif } @@ -289,9 +322,10 @@ bool TryInsert(const ClockHandleBasicData& proto, ClockHandle& h, return b; } +// Func must be const HandleImpl& -> void callable template -void ConstApplyToEntriesRange(Func /*const HandleImpl& -> void*/ func, - const HandleImpl* begin, const HandleImpl* end, +void ConstApplyToEntriesRange(const Func& func, const HandleImpl* begin, + const HandleImpl* end, bool apply_if_will_be_deleted) { uint64_t check_state_mask = ClockHandle::kStateShareableBit; if (!apply_if_will_be_deleted) { @@ -300,14 +334,13 @@ void ConstApplyToEntriesRange(Func /*const HandleImpl& -> void*/ func, for (const HandleImpl* h = begin; h < end; ++h) { // Note: to avoid using compare_exchange, we have to be extra careful. - uint64_t old_meta = h->meta.load(std::memory_order_relaxed); + uint64_t old_meta = h->meta.LoadRelaxed(); // Check if it's an entry visible to lookups if ((old_meta >> ClockHandle::kStateShift) & check_state_mask) { // Increment acquire counter. Note: it's possible that the entry has // completely changed since we loaded old_meta, but incrementing acquire // count is always safe. (Similar to optimistic Lookup here.) - old_meta = h->meta.fetch_add(ClockHandle::kAcquireIncrement, - std::memory_order_acquire); + old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement); // Check whether we actually acquired a reference. if ((old_meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit) { @@ -316,8 +349,7 @@ void ConstApplyToEntriesRange(Func /*const HandleImpl& -> void*/ func, func(*h); } // Pretend we never took the reference - h->meta.fetch_sub(ClockHandle::kAcquireIncrement, - std::memory_order_release); + Unref(*h); // No net change, so don't need to check for overflow } else { // For other states, incrementing the acquire counter has no effect @@ -329,6 +361,18 @@ void ConstApplyToEntriesRange(Func /*const HandleImpl& -> void*/ func, } } +constexpr uint32_t kStrictCapacityLimitBit = 1u << 31; + +uint32_t SanitizeEncodeEecAndScl(int eviction_effort_cap, + bool strict_capacit_limit) { + eviction_effort_cap = std::max(int{1}, eviction_effort_cap); + eviction_effort_cap = + std::min(static_cast(~kStrictCapacityLimitBit), eviction_effort_cap); + uint32_t eec_and_scl = static_cast(eviction_effort_cap); + eec_and_scl |= strict_capacit_limit ? kStrictCapacityLimitBit : 0; + return eec_and_scl; +} + } // namespace void ClockHandleBasicData::FreeData(MemoryAllocator* allocator) const { @@ -350,26 +394,28 @@ HandleImpl* BaseClockTable::StandaloneInsert( uint64_t meta = uint64_t{ClockHandle::kStateInvisible} << ClockHandle::kStateShift; meta |= uint64_t{1} << ClockHandle::kAcquireCounterShift; - h->meta.store(meta, std::memory_order_release); + h->meta.Store(meta); // Keep track of how much of usage is standalone - standalone_usage_.fetch_add(proto.GetTotalCharge(), - std::memory_order_relaxed); + standalone_usage_.FetchAddRelaxed(proto.GetTotalCharge()); return h; } template typename Table::HandleImpl* BaseClockTable::CreateStandalone( - ClockHandleBasicData& proto, size_t capacity, bool strict_capacity_limit, + ClockHandleBasicData& proto, size_t capacity, uint32_t eec_and_scl, bool allow_uncharged) { Table& derived = static_cast(*this); typename Table::InsertState state; derived.StartInsert(state); const size_t total_charge = proto.GetTotalCharge(); - if (strict_capacity_limit) { + // NOTE: we can use eec_and_scl as eviction_effort_cap below because + // strict_capacity_limit=true is supposed to disable the limit on eviction + // effort, and a large value effectively does that. + if (eec_and_scl & kStrictCapacityLimitBit) { Status s = ChargeUsageMaybeEvictStrict( total_charge, capacity, - /*need_evict_for_occupancy=*/false, state); + /*need_evict_for_occupancy=*/false, eec_and_scl, state); if (!s.ok()) { if (allow_uncharged) { proto.total_charge = 0; @@ -381,10 +427,10 @@ typename Table::HandleImpl* BaseClockTable::CreateStandalone( // Case strict_capacity_limit == false bool success = ChargeUsageMaybeEvictNonStrict
( total_charge, capacity, - /*need_evict_for_occupancy=*/false, state); + /*need_evict_for_occupancy=*/false, eec_and_scl, state); if (!success) { // Force the issue - usage_.fetch_add(total_charge, std::memory_order_relaxed); + usage_.FetchAddRelaxed(total_charge); } } @@ -394,23 +440,22 @@ typename Table::HandleImpl* BaseClockTable::CreateStandalone( template Status BaseClockTable::ChargeUsageMaybeEvictStrict( size_t total_charge, size_t capacity, bool need_evict_for_occupancy, - typename Table::InsertState& state) { + uint32_t eviction_effort_cap, typename Table::InsertState& state) { if (total_charge > capacity) { return Status::MemoryLimit( "Cache entry too large for a single cache shard: " + std::to_string(total_charge) + " > " + std::to_string(capacity)); } // Grab any available capacity, and free up any more required. - size_t old_usage = usage_.load(std::memory_order_relaxed); + size_t old_usage = usage_.LoadRelaxed(); size_t new_usage; - if (LIKELY(old_usage != capacity)) { - do { - new_usage = std::min(capacity, old_usage + total_charge); - } while (!usage_.compare_exchange_weak(old_usage, new_usage, - std::memory_order_relaxed)); - } else { - new_usage = old_usage; - } + do { + new_usage = std::min(capacity, old_usage + total_charge); + if (new_usage == old_usage) { + // No change needed + break; + } + } while (!usage_.CasWeakRelaxed(old_usage, new_usage)); // How much do we need to evict then? size_t need_evict_charge = old_usage + total_charge - new_usage; size_t request_evict_charge = need_evict_charge; @@ -419,22 +464,19 @@ Status BaseClockTable::ChargeUsageMaybeEvictStrict( request_evict_charge = 1; } if (request_evict_charge > 0) { - size_t evicted_charge = 0; - size_t evicted_count = 0; - static_cast(this)->Evict(request_evict_charge, &evicted_charge, - &evicted_count, state); - occupancy_.fetch_sub(evicted_count, std::memory_order_release); - if (LIKELY(evicted_charge > need_evict_charge)) { - assert(evicted_count > 0); + EvictionData data; + static_cast(this)->Evict(request_evict_charge, state, &data, + eviction_effort_cap); + occupancy_.FetchSub(data.freed_count); + if (LIKELY(data.freed_charge > need_evict_charge)) { + assert(data.freed_count > 0); // Evicted more than enough - usage_.fetch_sub(evicted_charge - need_evict_charge, - std::memory_order_relaxed); - } else if (evicted_charge < need_evict_charge || - (UNLIKELY(need_evict_for_occupancy) && evicted_count == 0)) { + usage_.FetchSubRelaxed(data.freed_charge - need_evict_charge); + } else if (data.freed_charge < need_evict_charge || + (UNLIKELY(need_evict_for_occupancy) && data.freed_count == 0)) { // Roll back to old usage minus evicted - usage_.fetch_sub(evicted_charge + (new_usage - old_usage), - std::memory_order_relaxed); - if (evicted_charge < need_evict_charge) { + usage_.FetchSubRelaxed(data.freed_charge + (new_usage - old_usage)); + if (data.freed_charge < need_evict_charge) { return Status::MemoryLimit( "Insert failed because unable to evict entries to stay within " "capacity limit."); @@ -446,7 +488,7 @@ Status BaseClockTable::ChargeUsageMaybeEvictStrict( } // If we needed to evict something and we are proceeding, we must have // evicted something. - assert(evicted_count > 0); + assert(data.freed_count > 0); } return Status::OK(); } @@ -454,7 +496,7 @@ Status BaseClockTable::ChargeUsageMaybeEvictStrict( template inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict( size_t total_charge, size_t capacity, bool need_evict_for_occupancy, - typename Table::InsertState& state) { + uint32_t eviction_effort_cap, typename Table::InsertState& state) { // For simplicity, we consider that either the cache can accept the insert // with no evictions, or we must evict enough to make (at least) enough // space. It could lead to unnecessary failures or excessive evictions in @@ -464,7 +506,7 @@ inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict( // charge. Thus, we should evict some extra if it's not a signifcant // portion of the shard capacity. This can have the side benefit of // involving fewer threads in eviction. - size_t old_usage = usage_.load(std::memory_order_relaxed); + size_t old_usage = usage_.LoadRelaxed(); size_t need_evict_charge; // NOTE: if total_charge > old_usage, there isn't yet enough to evict // `total_charge` amount. Even if we only try to evict `old_usage` amount, @@ -488,34 +530,59 @@ inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict( // deal with occupancy need_evict_charge = 1; } - size_t evicted_charge = 0; - size_t evicted_count = 0; + EvictionData data; if (need_evict_charge > 0) { - static_cast(this)->Evict(need_evict_charge, &evicted_charge, - &evicted_count, state); + static_cast(this)->Evict(need_evict_charge, state, &data, + eviction_effort_cap); // Deal with potential occupancy deficit - if (UNLIKELY(need_evict_for_occupancy) && evicted_count == 0) { - assert(evicted_charge == 0); + if (UNLIKELY(need_evict_for_occupancy) && data.freed_count == 0) { + assert(data.freed_charge == 0); // Can't meet occupancy requirement return false; } else { // Update occupancy for evictions - occupancy_.fetch_sub(evicted_count, std::memory_order_release); + occupancy_.FetchSub(data.freed_count); } } // Track new usage even if we weren't able to evict enough - usage_.fetch_add(total_charge - evicted_charge, std::memory_order_relaxed); + usage_.FetchAddRelaxed(total_charge - data.freed_charge); // No underflow - assert(usage_.load(std::memory_order_relaxed) < SIZE_MAX / 2); + assert(usage_.LoadRelaxed() < SIZE_MAX / 2); // Success return true; } +void BaseClockTable::TrackAndReleaseEvictedEntry(ClockHandle* h) { + bool took_value_ownership = false; + if (eviction_callback_) { + // For key reconstructed from hash + UniqueId64x2 unhashed; + took_value_ownership = + eviction_callback_(ClockCacheShard::ReverseHash( + h->GetHash(), &unhashed, hash_seed_), + reinterpret_cast(h), + h->meta.LoadRelaxed() & ClockHandle::kHitBitMask); + } + if (!took_value_ownership) { + h->FreeData(allocator_); + } + MarkEmpty(*h); +} + +bool IsEvictionEffortExceeded(const BaseClockTable::EvictionData& data, + uint32_t eviction_effort_cap) { + // Basically checks whether the ratio of useful effort to wasted effort is + // too low, with a start-up allowance for wasted effort before any useful + // effort. + return (data.freed_count + 1U) * uint64_t{eviction_effort_cap} <= + data.seen_pinned_count; +} + template Status BaseClockTable::Insert(const ClockHandleBasicData& proto, typename Table::HandleImpl** handle, Cache::Priority priority, size_t capacity, - bool strict_capacity_limit) { + uint32_t eec_and_scl) { using HandleImpl = typename Table::HandleImpl; Table& derived = static_cast(*this); @@ -524,7 +591,7 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto, // Do we have the available occupancy? Optimistically assume we do // and deal with it if we don't. - size_t old_occupancy = occupancy_.fetch_add(1, std::memory_order_acquire); + size_t old_occupancy = occupancy_.FetchAdd(1); // Whether we over-committed and need an eviction to make up for it bool need_evict_for_occupancy = !derived.GrowIfNeeded(old_occupancy + 1, state); @@ -533,21 +600,24 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto, // strict_capacity_limit, but mostly pessimistic. bool use_standalone_insert = false; const size_t total_charge = proto.GetTotalCharge(); - if (strict_capacity_limit) { + // NOTE: we can use eec_and_scl as eviction_effort_cap below because + // strict_capacity_limit=true is supposed to disable the limit on eviction + // effort, and a large value effectively does that. + if (eec_and_scl & kStrictCapacityLimitBit) { Status s = ChargeUsageMaybeEvictStrict
( - total_charge, capacity, need_evict_for_occupancy, state); + total_charge, capacity, need_evict_for_occupancy, eec_and_scl, state); if (!s.ok()) { // Revert occupancy - occupancy_.fetch_sub(1, std::memory_order_relaxed); + occupancy_.FetchSubRelaxed(1); return s; } } else { // Case strict_capacity_limit == false bool success = ChargeUsageMaybeEvictNonStrict
( - total_charge, capacity, need_evict_for_occupancy, state); + total_charge, capacity, need_evict_for_occupancy, eec_and_scl, state); if (!success) { // Revert occupancy - occupancy_.fetch_sub(1, std::memory_order_relaxed); + occupancy_.FetchSubRelaxed(1); if (handle == nullptr) { // Don't insert the entry but still return ok, as if the entry // inserted into cache and evicted immediately. @@ -555,7 +625,7 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto, return Status::OK(); } else { // Need to track usage of fallback standalone insert - usage_.fetch_add(total_charge, std::memory_order_relaxed); + usage_.FetchAddRelaxed(total_charge); use_standalone_insert = true; } } @@ -585,13 +655,13 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto, } // Not inserted // Revert occupancy - occupancy_.fetch_sub(1, std::memory_order_relaxed); + occupancy_.FetchSubRelaxed(1); // Maybe fall back on standalone insert if (handle == nullptr) { // Revert usage - usage_.fetch_sub(total_charge, std::memory_order_relaxed); + usage_.FetchSubRelaxed(total_charge); // No underflow - assert(usage_.load(std::memory_order_relaxed) < SIZE_MAX / 2); + assert(usage_.LoadRelaxed() < SIZE_MAX / 2); // As if unrefed entry immdiately evicted proto.FreeData(allocator_); return Status::OK(); @@ -615,8 +685,7 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto, void BaseClockTable::Ref(ClockHandle& h) { // Increment acquire counter - uint64_t old_meta = h.meta.fetch_add(ClockHandle::kAcquireIncrement, - std::memory_order_acquire); + uint64_t old_meta = h.meta.FetchAdd(ClockHandle::kAcquireIncrement); assert((old_meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit); @@ -628,8 +697,7 @@ void BaseClockTable::Ref(ClockHandle& h) { #ifndef NDEBUG void BaseClockTable::TEST_RefN(ClockHandle& h, size_t n) { // Increment acquire counter - uint64_t old_meta = h.meta.fetch_add(n * ClockHandle::kAcquireIncrement, - std::memory_order_acquire); + uint64_t old_meta = h.meta.FetchAdd(n * ClockHandle::kAcquireIncrement); assert((old_meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit); @@ -641,17 +709,16 @@ void BaseClockTable::TEST_ReleaseNMinus1(ClockHandle* h, size_t n) { // Like n-1 Releases, but assumes one more will happen in the caller to take // care of anything like erasing an unreferenced, invisible entry. - uint64_t old_meta = h->meta.fetch_add( - (n - 1) * ClockHandle::kReleaseIncrement, std::memory_order_acquire); + uint64_t old_meta = + h->meta.FetchAdd((n - 1) * ClockHandle::kReleaseIncrement); assert((old_meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit); (void)old_meta; } #endif -HyperClockTable::HyperClockTable( - size_t capacity, bool /*strict_capacity_limit*/, - CacheMetadataChargePolicy metadata_charge_policy, +FixedHyperClockTable::FixedHyperClockTable( + size_t capacity, CacheMetadataChargePolicy metadata_charge_policy, MemoryAllocator* allocator, const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed, const Opts& opts) @@ -665,25 +732,25 @@ HyperClockTable::HyperClockTable( array_(new HandleImpl[size_t{1} << length_bits_]) { if (metadata_charge_policy == CacheMetadataChargePolicy::kFullChargeCacheMetadata) { - usage_ += size_t{GetTableSize()} * sizeof(HandleImpl); + usage_.FetchAddRelaxed(size_t{GetTableSize()} * sizeof(HandleImpl)); } static_assert(sizeof(HandleImpl) == 64U, "Expecting size / alignment with common cache line size"); } -HyperClockTable::~HyperClockTable() { +FixedHyperClockTable::~FixedHyperClockTable() { // Assumes there are no references or active operations on any slot/element // in the table. for (size_t i = 0; i < GetTableSize(); i++) { HandleImpl& h = array_[i]; - switch (h.meta >> ClockHandle::kStateShift) { + switch (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) { case ClockHandle::kStateEmpty: // noop break; case ClockHandle::kStateInvisible: // rare but possible case ClockHandle::kStateVisible: - assert(GetRefcount(h.meta) == 0); + assert(GetRefcount(h.meta.LoadRelaxed()) == 0); h.FreeData(allocator_); #ifndef NDEBUG Rollback(h.hashed_key, &h); @@ -699,22 +766,22 @@ HyperClockTable::~HyperClockTable() { #ifndef NDEBUG for (size_t i = 0; i < GetTableSize(); i++) { - assert(array_[i].displacements.load() == 0); + assert(array_[i].displacements.LoadRelaxed() == 0); } #endif - assert(usage_.load() == 0 || - usage_.load() == size_t{GetTableSize()} * sizeof(HandleImpl)); - assert(occupancy_ == 0); + assert(usage_.LoadRelaxed() == 0 || + usage_.LoadRelaxed() == size_t{GetTableSize()} * sizeof(HandleImpl)); + assert(occupancy_.LoadRelaxed() == 0); } -void HyperClockTable::StartInsert(InsertState&) {} +void FixedHyperClockTable::StartInsert(InsertState&) {} -bool HyperClockTable::GrowIfNeeded(size_t new_occupancy, InsertState&) { +bool FixedHyperClockTable::GrowIfNeeded(size_t new_occupancy, InsertState&) { return new_occupancy <= occupancy_limit_; } -HyperClockTable::HandleImpl* HyperClockTable::DoInsert( +FixedHyperClockTable::HandleImpl* FixedHyperClockTable::DoInsert( const ClockHandleBasicData& proto, uint64_t initial_countdown, bool keep_ref, InsertState&) { bool already_matches = false; @@ -739,7 +806,7 @@ HyperClockTable::HandleImpl* HyperClockTable::DoInsert( // Search is ending. Roll back displacements Rollback(proto.hashed_key, h); } else { - h->displacements.fetch_add(1, std::memory_order_relaxed); + h->displacements.FetchAddRelaxed(1); } }); if (already_matches) { @@ -761,20 +828,20 @@ HyperClockTable::HandleImpl* HyperClockTable::DoInsert( return nullptr; } -HyperClockTable::HandleImpl* HyperClockTable::Lookup( +FixedHyperClockTable::HandleImpl* FixedHyperClockTable::Lookup( const UniqueId64x2& hashed_key) { HandleImpl* e = FindSlot( hashed_key, [&](HandleImpl* h) { // Mostly branch-free version (similar performance) /* - uint64_t old_meta = h->meta.fetch_add(ClockHandle::kAcquireIncrement, + uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement, std::memory_order_acquire); bool Shareable = (old_meta >> (ClockHandle::kStateShift + 1)) & 1U; bool visible = (old_meta >> ClockHandle::kStateShift) & 1U; bool match = (h->key == key) & visible; - h->meta.fetch_sub(static_cast(Shareable & !match) << - ClockHandle::kAcquireCounterShift, std::memory_order_release); return + h->meta.FetchSub(static_cast(Shareable & !match) << + ClockHandle::kAcquireCounterShift); return match; */ // Optimistic lookup should pay off when the table is relatively @@ -782,53 +849,49 @@ HyperClockTable::HandleImpl* HyperClockTable::Lookup( constexpr bool kOptimisticLookup = true; uint64_t old_meta; if (!kOptimisticLookup) { - old_meta = h->meta.load(std::memory_order_acquire); + old_meta = h->meta.Load(); if ((old_meta >> ClockHandle::kStateShift) != ClockHandle::kStateVisible) { return false; } } // (Optimistically) increment acquire counter - old_meta = h->meta.fetch_add(ClockHandle::kAcquireIncrement, - std::memory_order_acquire); + old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement); // Check if it's an entry visible to lookups if ((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateVisible) { // Acquired a read reference if (h->hashed_key == hashed_key) { // Match + // Update the hit bit + if (eviction_callback_) { + h->meta.FetchOrRelaxed(uint64_t{1} << ClockHandle::kHitBitShift); + } return true; } else { // Mismatch. Pretend we never took the reference - old_meta = h->meta.fetch_sub(ClockHandle::kAcquireIncrement, - std::memory_order_release); + Unref(*h); } } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateInvisible)) { // Pretend we never took the reference - // WART: there's a tiny chance we release last ref to invisible - // entry here. If that happens, we let eviction take care of it. - old_meta = h->meta.fetch_sub(ClockHandle::kAcquireIncrement, - std::memory_order_release); + Unref(*h); } else { // For other states, incrementing the acquire counter has no effect // so we don't need to undo it. Furthermore, we cannot safely undo // it because we did not acquire a read reference to lock the // entry in a Shareable state. } - (void)old_meta; return false; }, - [&](HandleImpl* h) { - return h->displacements.load(std::memory_order_relaxed) == 0; - }, + [&](HandleImpl* h) { return h->displacements.LoadRelaxed() == 0; }, [&](HandleImpl* /*h*/, bool /*is_last*/) {}); return e; } -bool HyperClockTable::Release(HandleImpl* h, bool useful, - bool erase_if_last_ref) { +bool FixedHyperClockTable::Release(HandleImpl* h, bool useful, + bool erase_if_last_ref) { // In contrast with LRUCache's Release, this function won't delete the handle // when the cache is above capacity and the reference is the last one. Space // is only freed up by EvictFromClock (called by Insert when space is needed) @@ -837,12 +900,10 @@ bool HyperClockTable::Release(HandleImpl* h, bool useful, uint64_t old_meta; if (useful) { // Increment release counter to indicate was used - old_meta = h->meta.fetch_add(ClockHandle::kReleaseIncrement, - std::memory_order_release); + old_meta = h->meta.FetchAdd(ClockHandle::kReleaseIncrement); } else { // Decrement acquire counter to pretend it never happened - old_meta = h->meta.fetch_sub(ClockHandle::kAcquireIncrement, - std::memory_order_release); + old_meta = h->meta.FetchSub(ClockHandle::kAcquireIncrement); } assert((old_meta >> ClockHandle::kStateShift) & @@ -855,7 +916,10 @@ bool HyperClockTable::Release(HandleImpl* h, bool useful, if (erase_if_last_ref || UNLIKELY(old_meta >> ClockHandle::kStateShift == ClockHandle::kStateInvisible)) { - // Update for last fetch_add op + // FIXME: There's a chance here that another thread could replace this + // entry and we end up erasing the wrong one. + + // Update for last FetchAdd op if (useful) { old_meta += ClockHandle::kReleaseIncrement; } else { @@ -877,18 +941,17 @@ bool HyperClockTable::Release(HandleImpl* h, bool useful, // Note that there's a small chance that we release, another thread // replaces this entry with another, reaches zero refs, and then we end // up erasing that other entry. That's an acceptable risk / imprecision. - } while (!h->meta.compare_exchange_weak( - old_meta, - uint64_t{ClockHandle::kStateConstruction} << ClockHandle::kStateShift, - std::memory_order_acquire)); + } while ( + !h->meta.CasWeak(old_meta, uint64_t{ClockHandle::kStateConstruction} + << ClockHandle::kStateShift)); // Took ownership size_t total_charge = h->GetTotalCharge(); if (UNLIKELY(h->IsStandalone())) { h->FreeData(allocator_); // Delete standalone handle delete h; - standalone_usage_.fetch_sub(total_charge, std::memory_order_relaxed); - usage_.fetch_sub(total_charge, std::memory_order_relaxed); + standalone_usage_.FetchSubRelaxed(total_charge); + usage_.FetchSubRelaxed(total_charge); } else { Rollback(h->hashed_key, h); FreeDataMarkEmpty(*h, allocator_); @@ -903,7 +966,7 @@ bool HyperClockTable::Release(HandleImpl* h, bool useful, } #ifndef NDEBUG -void HyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) { +void FixedHyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) { if (n > 0) { // Do n-1 simple releases first TEST_ReleaseNMinus1(h, n); @@ -914,14 +977,13 @@ void HyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) { } #endif -void HyperClockTable::Erase(const UniqueId64x2& hashed_key) { +void FixedHyperClockTable::Erase(const UniqueId64x2& hashed_key) { (void)FindSlot( hashed_key, [&](HandleImpl* h) { // Could be multiple entries in rare cases. Erase them all. // Optimistically increment acquire counter - uint64_t old_meta = h->meta.fetch_add(ClockHandle::kAcquireIncrement, - std::memory_order_acquire); + uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement); // Check if it's an entry visible to lookups if ((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateVisible) { @@ -929,9 +991,8 @@ void HyperClockTable::Erase(const UniqueId64x2& hashed_key) { if (h->hashed_key == hashed_key) { // Match. Set invisible. old_meta = - h->meta.fetch_and(~(uint64_t{ClockHandle::kStateVisibleBit} - << ClockHandle::kStateShift), - std::memory_order_acq_rel); + h->meta.FetchAnd(~(uint64_t{ClockHandle::kStateVisibleBit} + << ClockHandle::kStateShift)); // Apply update to local copy old_meta &= ~(uint64_t{ClockHandle::kStateVisibleBit} << ClockHandle::kStateShift); @@ -941,14 +1002,11 @@ void HyperClockTable::Erase(const UniqueId64x2& hashed_key) { if (refcount > 1) { // Not last ref at some point in time during this Erase call // Pretend we never took the reference - h->meta.fetch_sub(ClockHandle::kAcquireIncrement, - std::memory_order_release); + Unref(*h); break; - } else if (h->meta.compare_exchange_weak( - old_meta, - uint64_t{ClockHandle::kStateConstruction} - << ClockHandle::kStateShift, - std::memory_order_acq_rel)) { + } else if (h->meta.CasWeak( + old_meta, uint64_t{ClockHandle::kStateConstruction} + << ClockHandle::kStateShift)) { // Took ownership assert(hashed_key == h->hashed_key); size_t total_charge = h->GetTotalCharge(); @@ -962,40 +1020,32 @@ void HyperClockTable::Erase(const UniqueId64x2& hashed_key) { } } else { // Mismatch. Pretend we never took the reference - h->meta.fetch_sub(ClockHandle::kAcquireIncrement, - std::memory_order_release); + Unref(*h); } } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateInvisible)) { // Pretend we never took the reference - // WART: there's a tiny chance we release last ref to invisible - // entry here. If that happens, we let eviction take care of it. - h->meta.fetch_sub(ClockHandle::kAcquireIncrement, - std::memory_order_release); + Unref(*h); } else { // For other states, incrementing the acquire counter has no effect // so we don't need to undo it. } return false; }, - [&](HandleImpl* h) { - return h->displacements.load(std::memory_order_relaxed) == 0; - }, + [&](HandleImpl* h) { return h->displacements.LoadRelaxed() == 0; }, [&](HandleImpl* /*h*/, bool /*is_last*/) {}); } -void HyperClockTable::EraseUnRefEntries() { +void FixedHyperClockTable::EraseUnRefEntries() { for (size_t i = 0; i <= this->length_bits_mask_; i++) { HandleImpl& h = array_[i]; - uint64_t old_meta = h.meta.load(std::memory_order_relaxed); + uint64_t old_meta = h.meta.LoadRelaxed(); if (old_meta & (uint64_t{ClockHandle::kStateShareableBit} << ClockHandle::kStateShift) && GetRefcount(old_meta) == 0 && - h.meta.compare_exchange_strong(old_meta, - uint64_t{ClockHandle::kStateConstruction} - << ClockHandle::kStateShift, - std::memory_order_acquire)) { + h.meta.CasStrong(old_meta, uint64_t{ClockHandle::kStateConstruction} + << ClockHandle::kStateShift)) { // Took ownership size_t total_charge = h.GetTotalCharge(); Rollback(h.hashed_key, &h); @@ -1006,9 +1056,9 @@ void HyperClockTable::EraseUnRefEntries() { } template -inline HyperClockTable::HandleImpl* HyperClockTable::FindSlot( - const UniqueId64x2& hashed_key, MatchFn match_fn, AbortFn abort_fn, - UpdateFn update_fn) { +inline FixedHyperClockTable::HandleImpl* FixedHyperClockTable::FindSlot( + const UniqueId64x2& hashed_key, const MatchFn& match_fn, + const AbortFn& abort_fn, const UpdateFn& update_fn) { // NOTE: upper 32 bits of hashed_key[0] is used for sharding // // We use double-hashing probing. Every probe in the sequence is a @@ -1041,30 +1091,30 @@ inline HyperClockTable::HandleImpl* HyperClockTable::FindSlot( return nullptr; } -inline void HyperClockTable::Rollback(const UniqueId64x2& hashed_key, - const HandleImpl* h) { +inline void FixedHyperClockTable::Rollback(const UniqueId64x2& hashed_key, + const HandleImpl* h) { size_t current = ModTableSize(hashed_key[1]); size_t increment = static_cast(hashed_key[0]) | 1U; while (&array_[current] != h) { - array_[current].displacements.fetch_sub(1, std::memory_order_relaxed); + array_[current].displacements.FetchSubRelaxed(1); current = ModTableSize(current + increment); } } -inline void HyperClockTable::ReclaimEntryUsage(size_t total_charge) { - auto old_occupancy = occupancy_.fetch_sub(1U, std::memory_order_release); +inline void FixedHyperClockTable::ReclaimEntryUsage(size_t total_charge) { + auto old_occupancy = occupancy_.FetchSub(1U); (void)old_occupancy; // No underflow assert(old_occupancy > 0); - auto old_usage = usage_.fetch_sub(total_charge, std::memory_order_relaxed); + auto old_usage = usage_.FetchSubRelaxed(total_charge); (void)old_usage; // No underflow assert(old_usage >= total_charge); } -inline void HyperClockTable::Evict(size_t requested_charge, - size_t* freed_charge, size_t* freed_count, - InsertState&) { +inline void FixedHyperClockTable::Evict(size_t requested_charge, InsertState&, + EvictionData* data, + uint32_t eviction_effort_cap) { // precondition assert(requested_charge > 0); @@ -1072,8 +1122,7 @@ inline void HyperClockTable::Evict(size_t requested_charge, constexpr size_t step_size = 4; // First (concurrent) increment clock pointer - uint64_t old_clock_pointer = - clock_pointer_.fetch_add(step_size, std::memory_order_relaxed); + uint64_t old_clock_pointer = clock_pointer_.FetchAddRelaxed(step_size); // Cap the eviction effort at this thread (along with those operating in // parallel) circling through the whole structure kMaxCountdown times. @@ -1083,42 +1132,30 @@ inline void HyperClockTable::Evict(size_t requested_charge, uint64_t max_clock_pointer = old_clock_pointer + (ClockHandle::kMaxCountdown << length_bits_); - // For key reconstructed from hash - UniqueId64x2 unhashed; - for (;;) { for (size_t i = 0; i < step_size; i++) { HandleImpl& h = array_[ModTableSize(Lower32of64(old_clock_pointer + i))]; - bool evicting = ClockUpdate(h); + bool evicting = ClockUpdate(h, data); if (evicting) { Rollback(h.hashed_key, &h); - *freed_charge += h.GetTotalCharge(); - *freed_count += 1; - bool took_ownership = false; - if (eviction_callback_) { - took_ownership = - eviction_callback_(ClockCacheShard::ReverseHash( - h.GetHash(), &unhashed, hash_seed_), - reinterpret_cast(&h)); - } - if (!took_ownership) { - h.FreeData(allocator_); - } - MarkEmpty(h); + TrackAndReleaseEvictedEntry(&h); } } // Loop exit condition - if (*freed_charge >= requested_charge) { + if (data->freed_charge >= requested_charge) { return; } if (old_clock_pointer >= max_clock_pointer) { return; } + if (IsEvictionEffortExceeded(*data, eviction_effort_cap)) { + eviction_effort_exceeded_count_.FetchAddRelaxed(1); + return; + } // Advance clock pointer (concurrently) - old_clock_pointer = - clock_pointer_.fetch_add(step_size, std::memory_order_relaxed); + old_clock_pointer = clock_pointer_.FetchAddRelaxed(step_size); } } @@ -1130,12 +1167,14 @@ ClockCacheShard
::ClockCacheShard( const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed, const typename Table::Opts& opts) : CacheShardBase(metadata_charge_policy), - table_(capacity, strict_capacity_limit, metadata_charge_policy, allocator, - eviction_callback, hash_seed, opts), + table_(capacity, metadata_charge_policy, allocator, eviction_callback, + hash_seed, opts), capacity_(capacity), - strict_capacity_limit_(strict_capacity_limit) { + eec_and_scl_(SanitizeEncodeEecAndScl(opts.eviction_effort_cap, + strict_capacity_limit)) { // Initial charge metadata should not exceed capacity - assert(table_.GetUsage() <= capacity_ || capacity_ < sizeof(HandleImpl)); + assert(table_.GetUsage() <= capacity_.LoadRelaxed() || + capacity_.LoadRelaxed() < sizeof(HandleImpl)); } template @@ -1177,7 +1216,7 @@ void ClockCacheShard
::ApplyToSomeEntries( table_.HandlePtr(index_begin), table_.HandlePtr(index_end), false); } -int HyperClockTable::CalcHashBits( +int FixedHyperClockTable::CalcHashBits( size_t capacity, size_t estimated_value_size, CacheMetadataChargePolicy metadata_charge_policy) { double average_slot_charge = estimated_value_size * kLoadFactor; @@ -1201,15 +1240,18 @@ int HyperClockTable::CalcHashBits( template void ClockCacheShard
::SetCapacity(size_t capacity) { - capacity_.store(capacity, std::memory_order_relaxed); + capacity_.StoreRelaxed(capacity); // next Insert will take care of any necessary evictions } template void ClockCacheShard
::SetStrictCapacityLimit( bool strict_capacity_limit) { - strict_capacity_limit_.store(strict_capacity_limit, - std::memory_order_relaxed); + if (strict_capacity_limit) { + eec_and_scl_.FetchOrRelaxed(kStrictCapacityLimitBit); + } else { + eec_and_scl_.FetchAndRelaxed(~kStrictCapacityLimitBit); + } // next Insert will take care of any necessary evictions } @@ -1229,9 +1271,9 @@ Status ClockCacheShard
::Insert(const Slice& key, proto.value = value; proto.helper = helper; proto.total_charge = charge; - return table_.template Insert
( - proto, handle, priority, capacity_.load(std::memory_order_relaxed), - strict_capacity_limit_.load(std::memory_order_relaxed)); + return table_.template Insert
(proto, handle, priority, + capacity_.LoadRelaxed(), + eec_and_scl_.LoadRelaxed()); } template @@ -1246,9 +1288,9 @@ typename Table::HandleImpl* ClockCacheShard
::CreateStandalone( proto.value = obj; proto.helper = helper; proto.total_charge = charge; - return table_.template CreateStandalone
( - proto, capacity_.load(std::memory_order_relaxed), - strict_capacity_limit_.load(std::memory_order_relaxed), allow_uncharged); + return table_.template CreateStandalone
(proto, capacity_.LoadRelaxed(), + eec_and_scl_.LoadRelaxed(), + allow_uncharged); } template @@ -1317,7 +1359,7 @@ size_t ClockCacheShard
::GetStandaloneUsage() const { template size_t ClockCacheShard
::GetCapacity() const { - return capacity_; + return capacity_.LoadRelaxed(); } template @@ -1333,7 +1375,7 @@ size_t ClockCacheShard
::GetPinnedUsage() const { metadata_charge_policy_ == kFullChargeCacheMetadata; ConstApplyToEntriesRange( [&table_pinned_usage, charge_metadata](const HandleImpl& h) { - uint64_t meta = h.meta.load(std::memory_order_relaxed); + uint64_t meta = h.meta.LoadRelaxed(); uint64_t refcount = GetRefcount(meta); // Holding one ref for ConstApplyToEntriesRange assert(refcount > 0); @@ -1365,36 +1407,40 @@ size_t ClockCacheShard
::GetTableAddressCount() const { } // Explicit instantiation -template class ClockCacheShard; +template class ClockCacheShard; +template class ClockCacheShard; -HyperClockCache::HyperClockCache(const HyperClockCacheOptions& opts) - : ShardedCache(opts) { - assert(opts.estimated_entry_charge > 0 || - opts.metadata_charge_policy != kDontChargeCacheMetadata); +template +BaseHyperClockCache
::BaseHyperClockCache( + const HyperClockCacheOptions& opts) + : ShardedCache>(opts) { // TODO: should not need to go through two levels of pointer indirection to // get to table entries - size_t per_shard = GetPerShardCapacity(); + size_t per_shard = this->GetPerShardCapacity(); MemoryAllocator* alloc = this->memory_allocator(); - InitShards([&](Shard* cs) { - HyperClockTable::Opts table_opts; - table_opts.estimated_value_size = opts.estimated_entry_charge; + this->InitShards([&](Shard* cs) { + typename Table::Opts table_opts{opts}; new (cs) Shard(per_shard, opts.strict_capacity_limit, - opts.metadata_charge_policy, alloc, &eviction_callback_, - &hash_seed_, table_opts); + opts.metadata_charge_policy, alloc, + &this->eviction_callback_, &this->hash_seed_, table_opts); }); } -Cache::ObjectPtr HyperClockCache::Value(Handle* handle) { - return reinterpret_cast(handle)->value; +template +Cache::ObjectPtr BaseHyperClockCache
::Value(Handle* handle) { + return reinterpret_cast(handle)->value; } -size_t HyperClockCache::GetCharge(Handle* handle) const { - return reinterpret_cast(handle)->GetTotalCharge(); +template +size_t BaseHyperClockCache
::GetCharge(Handle* handle) const { + return reinterpret_cast(handle) + ->GetTotalCharge(); } -const Cache::CacheItemHelper* HyperClockCache::GetCacheItemHelper( +template +const Cache::CacheItemHelper* BaseHyperClockCache
::GetCacheItemHelper( Handle* handle) const { - auto h = reinterpret_cast(handle); + auto h = reinterpret_cast(handle); return h->helper; } @@ -1407,7 +1453,7 @@ namespace { // or actual occupancy very close to limit (>95% of limit). // Also, for each shard compute the recommended estimated_entry_charge, // and keep the minimum one for use as overall recommendation. -void AddShardEvaluation(const HyperClockCache::Shard& shard, +void AddShardEvaluation(const FixedHyperClockCache::Shard& shard, std::vector& predicted_load_factors, size_t& min_recommendation) { size_t usage = shard.GetUsage() - shard.GetStandaloneUsage(); @@ -1425,7 +1471,7 @@ void AddShardEvaluation(const HyperClockCache::Shard& shard, // If filled to capacity, what would the occupancy ratio be? double ratio = occ_ratio / usage_ratio; // Given max load factor, what that load factor be? - double lf = ratio * kStrictLoadFactor; + double lf = ratio * FixedHyperClockTable::kStrictLoadFactor; predicted_load_factors.push_back(lf); // Update min_recommendation also @@ -1433,17 +1479,96 @@ void AddShardEvaluation(const HyperClockCache::Shard& shard, min_recommendation = std::min(min_recommendation, recommendation); } +bool IsSlotOccupied(const ClockHandle& h) { + return (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) != 0; +} } // namespace -void HyperClockCache::ReportProblems( +// NOTE: GCC might warn about subobject linkage if this is in anon namespace +template +class LoadVarianceStats { + public: + std::string Report() const { + return "Overall " + PercentStr(positive_count_, samples_) + " (" + + std::to_string(positive_count_) + "/" + std::to_string(samples_) + + "), Min/Max/Window = " + PercentStr(min_, N) + "/" + + PercentStr(max_, N) + "/" + std::to_string(N) + + ", MaxRun{Pos/Neg} = " + std::to_string(max_pos_run_) + "/" + + std::to_string(max_neg_run_); + } + + void Add(bool positive) { + recent_[samples_ % N] = positive; + if (positive) { + ++positive_count_; + ++cur_pos_run_; + max_pos_run_ = std::max(max_pos_run_, cur_pos_run_); + cur_neg_run_ = 0; + } else { + ++cur_neg_run_; + max_neg_run_ = std::max(max_neg_run_, cur_neg_run_); + cur_pos_run_ = 0; + } + ++samples_; + if (samples_ >= N) { + size_t count_set = recent_.count(); + max_ = std::max(max_, count_set); + min_ = std::min(min_, count_set); + } + } + + private: + size_t max_ = 0; + size_t min_ = N; + size_t positive_count_ = 0; + size_t samples_ = 0; + size_t max_pos_run_ = 0; + size_t cur_pos_run_ = 0; + size_t max_neg_run_ = 0; + size_t cur_neg_run_ = 0; + std::bitset recent_; + + static std::string PercentStr(size_t a, size_t b) { + if (b == 0) { + return "??%"; + } else { + return std::to_string(uint64_t{100} * a / b) + "%"; + } + } +}; + +template +void BaseHyperClockCache
::ReportProblems( + const std::shared_ptr& info_log) const { + if (info_log->GetInfoLogLevel() <= InfoLogLevel::DEBUG_LEVEL) { + LoadVarianceStats slot_stats; + uint64_t eviction_effort_exceeded_count = 0; + this->ForEachShard([&](const BaseHyperClockCache
::Shard* shard) { + size_t count = shard->GetTableAddressCount(); + for (size_t i = 0; i < count; ++i) { + slot_stats.Add(IsSlotOccupied(*shard->GetTable().HandlePtr(i))); + } + eviction_effort_exceeded_count += + shard->GetTable().GetEvictionEffortExceededCount(); + }); + ROCKS_LOG_AT_LEVEL(info_log, InfoLogLevel::DEBUG_LEVEL, + "Slot occupancy stats: %s", slot_stats.Report().c_str()); + ROCKS_LOG_AT_LEVEL(info_log, InfoLogLevel::DEBUG_LEVEL, + "Eviction effort exceeded: %" PRIu64, + eviction_effort_exceeded_count); + } +} + +void FixedHyperClockCache::ReportProblems( const std::shared_ptr& info_log) const { + BaseHyperClockCache::ReportProblems(info_log); + uint32_t shard_count = GetNumShards(); std::vector predicted_load_factors; size_t min_recommendation = SIZE_MAX; - const_cast(this)->ForEachShard( - [&](HyperClockCache::Shard* shard) { - AddShardEvaluation(*shard, predicted_load_factors, min_recommendation); - }); + ForEachShard([&](const FixedHyperClockCache::Shard* shard) { + AddShardEvaluation(*shard, predicted_load_factors, min_recommendation); + }); if (predicted_load_factors.empty()) { // None operating "at limit" -> nothing to report @@ -1464,17 +1589,19 @@ void HyperClockCache::ReportProblems( predicted_load_factors.end(), 0.0) / shard_count; - constexpr double kLowSpecLoadFactor = kLoadFactor / 2; - constexpr double kMidSpecLoadFactor = kLoadFactor / 1.414; - if (average_load_factor > kLoadFactor) { + constexpr double kLowSpecLoadFactor = FixedHyperClockTable::kLoadFactor / 2; + constexpr double kMidSpecLoadFactor = + FixedHyperClockTable::kLoadFactor / 1.414; + if (average_load_factor > FixedHyperClockTable::kLoadFactor) { // Out of spec => Consider reporting load factor too high // Estimate effective overall capacity loss due to enforcing occupancy limit double lost_portion = 0.0; int over_count = 0; for (double lf : predicted_load_factors) { - if (lf > kStrictLoadFactor) { + if (lf > FixedHyperClockTable::kStrictLoadFactor) { ++over_count; - lost_portion += (lf - kStrictLoadFactor) / lf / shard_count; + lost_portion += + (lf - FixedHyperClockTable::kStrictLoadFactor) / lf / shard_count; } } // >= 20% loss -> error @@ -1498,10 +1625,10 @@ void HyperClockCache::ReportProblems( if (report) { ROCKS_LOG_AT_LEVEL( info_log, level, - "HyperClockCache@%p unable to use estimated %.1f%% capacity because " - "of " - "full occupancy in %d/%u cache shards (estimated_entry_charge too " - "high). Recommend estimated_entry_charge=%zu", + "FixedHyperClockCache@%p unable to use estimated %.1f%% capacity " + "because of full occupancy in %d/%u cache shards " + "(estimated_entry_charge too high). " + "Recommend estimated_entry_charge=%zu", this, lost_portion * 100.0, over_count, (unsigned)shard_count, min_recommendation); } @@ -1519,41 +1646,2007 @@ void HyperClockCache::ReportProblems( } ROCKS_LOG_AT_LEVEL( info_log, level, - "HyperClockCache@%p table has low occupancy at full capacity. Higher " - "estimated_entry_charge (about %.1fx) would likely improve " + "FixedHyperClockCache@%p table has low occupancy at full capacity. " + "Higher estimated_entry_charge (about %.1fx) would likely improve " "performance. Recommend estimated_entry_charge=%zu", this, kMidSpecLoadFactor / average_load_factor, min_recommendation); } } } -} // namespace clock_cache +// ======================================================================= +// AutoHyperClockCache +// ======================================================================= -// DEPRECATED (see public API) -std::shared_ptr NewClockCache( - size_t capacity, int num_shard_bits, bool strict_capacity_limit, - CacheMetadataChargePolicy metadata_charge_policy) { - return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit, - /* high_pri_pool_ratio */ 0.5, nullptr, - kDefaultToAdaptiveMutex, metadata_charge_policy, - /* low_pri_pool_ratio */ 0.0); +// See AutoHyperClockTable::length_info_ etc. for how the linear hashing +// metadata is encoded. Here are some example values: +// +// Used length | min shift | threshold | max shift +// 2 | 1 | 0 | 1 +// 3 | 1 | 1 | 2 +// 4 | 2 | 0 | 2 +// 5 | 2 | 1 | 3 +// 6 | 2 | 2 | 3 +// 7 | 2 | 3 | 3 +// 8 | 3 | 0 | 3 +// 9 | 3 | 1 | 4 +// ... +// Note: +// * min shift = floor(log2(used length)) +// * max shift = ceil(log2(used length)) +// * used length == (1 << shift) + threshold +// Also, shift=0 is never used in practice, so is reserved for "unset" + +namespace { + +inline int LengthInfoToMinShift(uint64_t length_info) { + int mask_shift = BitwiseAnd(length_info, int{255}); + assert(mask_shift <= 63); + assert(mask_shift > 0); + return mask_shift; } -std::shared_ptr HyperClockCacheOptions::MakeSharedCache() const { - // For sanitized options - HyperClockCacheOptions opts = *this; - if (opts.num_shard_bits >= 20) { - return nullptr; // The cache cannot be sharded into too many fine pieces. +inline size_t LengthInfoToThreshold(uint64_t length_info) { + return static_cast(length_info >> 8); +} + +inline size_t LengthInfoToUsedLength(uint64_t length_info) { + size_t threshold = LengthInfoToThreshold(length_info); + int shift = LengthInfoToMinShift(length_info); + assert(threshold < (size_t{1} << shift)); + size_t used_length = (size_t{1} << shift) + threshold; + assert(used_length >= 2); + return used_length; +} + +inline uint64_t UsedLengthToLengthInfo(size_t used_length) { + assert(used_length >= 2); + int shift = FloorLog2(used_length); + uint64_t threshold = BottomNBits(used_length, shift); + uint64_t length_info = + (uint64_t{threshold} << 8) + static_cast(shift); + assert(LengthInfoToUsedLength(length_info) == used_length); + assert(LengthInfoToMinShift(length_info) == shift); + assert(LengthInfoToThreshold(length_info) == threshold); + return length_info; +} + +inline size_t GetStartingLength(size_t capacity) { + if (capacity > port::kPageSize) { + // Start with one memory page + return port::kPageSize / sizeof(AutoHyperClockTable::HandleImpl); + } else { + // Mostly to make unit tests happy + return 4; } - if (opts.num_shard_bits < 0) { - // Use larger shard size to reduce risk of large entries clustering - // or skewing individual shards. - constexpr size_t min_shard_size = 32U * 1024U * 1024U; - opts.num_shard_bits = - GetDefaultCacheShardBits(opts.capacity, min_shard_size); +} + +inline size_t GetHomeIndex(uint64_t hash, int shift) { + return static_cast(BottomNBits(hash, shift)); +} + +inline void GetHomeIndexAndShift(uint64_t length_info, uint64_t hash, + size_t* home, int* shift) { + int min_shift = LengthInfoToMinShift(length_info); + size_t threshold = LengthInfoToThreshold(length_info); + bool extra_shift = GetHomeIndex(hash, min_shift) < threshold; + *home = GetHomeIndex(hash, min_shift + extra_shift); + *shift = min_shift + extra_shift; + assert(*home < LengthInfoToUsedLength(length_info)); +} + +inline int GetShiftFromNextWithShift(uint64_t next_with_shift) { + return BitwiseAnd(next_with_shift, + AutoHyperClockTable::HandleImpl::kShiftMask); +} + +inline size_t GetNextFromNextWithShift(uint64_t next_with_shift) { + return static_cast(next_with_shift >> + AutoHyperClockTable::HandleImpl::kNextShift); +} + +inline uint64_t MakeNextWithShift(size_t next, int shift) { + return (uint64_t{next} << AutoHyperClockTable::HandleImpl::kNextShift) | + static_cast(shift); +} + +inline uint64_t MakeNextWithShiftEnd(size_t head, int shift) { + return AutoHyperClockTable::HandleImpl::kNextEndFlags | + MakeNextWithShift(head, shift); +} + +// Helper function for Lookup +inline bool MatchAndRef(const UniqueId64x2* hashed_key, const ClockHandle& h, + int shift = 0, size_t home = 0, + bool* full_match_or_unknown = nullptr) { + // Must be at least something to match + assert(hashed_key || shift > 0); + + uint64_t old_meta; + // (Optimistically) increment acquire counter. + old_meta = h.meta.FetchAdd(ClockHandle::kAcquireIncrement); + // Check if it's a referencable (sharable) entry + if ((old_meta & (uint64_t{ClockHandle::kStateShareableBit} + << ClockHandle::kStateShift)) == 0) { + // For non-sharable states, incrementing the acquire counter has no effect + // so we don't need to undo it. Furthermore, we cannot safely undo + // it because we did not acquire a read reference to lock the + // entry in a Shareable state. + if (full_match_or_unknown) { + *full_match_or_unknown = true; + } + return false; + } + // Else acquired a read reference + assert(GetRefcount(old_meta + ClockHandle::kAcquireIncrement) > 0); + if (hashed_key && h.hashed_key == *hashed_key && + LIKELY(old_meta & (uint64_t{ClockHandle::kStateVisibleBit} + << ClockHandle::kStateShift))) { + // Match on full key, visible + if (full_match_or_unknown) { + *full_match_or_unknown = true; + } + return true; + } else if (shift > 0 && home == BottomNBits(h.hashed_key[1], shift)) { + // NOTE: upper 32 bits of hashed_key[0] is used for sharding + // Match on home address, possibly invisible + if (full_match_or_unknown) { + *full_match_or_unknown = false; + } + return true; + } else { + // Mismatch. Pretend we never took the reference + Unref(h); + if (full_match_or_unknown) { + *full_match_or_unknown = false; + } + return false; + } +} + +// Assumes a chain rewrite lock prevents concurrent modification of +// these chain pointers +void UpgradeShiftsOnRange(AutoHyperClockTable::HandleImpl* arr, + size_t& frontier, uint64_t stop_before_or_new_tail, + int old_shift, int new_shift) { + assert(frontier != SIZE_MAX); + assert(new_shift == old_shift + 1); + (void)old_shift; + (void)new_shift; + using HandleImpl = AutoHyperClockTable::HandleImpl; + for (;;) { + uint64_t next_with_shift = arr[frontier].chain_next_with_shift.Load(); + assert(GetShiftFromNextWithShift(next_with_shift) == old_shift); + if (next_with_shift == stop_before_or_new_tail) { + // Stopping at entry with pointer matching "stop before" + assert(!HandleImpl::IsEnd(next_with_shift)); + return; + } + if (HandleImpl::IsEnd(next_with_shift)) { + // Also update tail to new tail + assert(HandleImpl::IsEnd(stop_before_or_new_tail)); + arr[frontier].chain_next_with_shift.Store(stop_before_or_new_tail); + // Mark nothing left to upgrade + frontier = SIZE_MAX; + return; + } + // Next is another entry to process, so upgrade and advance frontier + arr[frontier].chain_next_with_shift.FetchAdd(1U); + assert(GetShiftFromNextWithShift(next_with_shift + 1) == new_shift); + frontier = GetNextFromNextWithShift(next_with_shift); + } +} + +size_t CalcOccupancyLimit(size_t used_length) { + return static_cast(used_length * AutoHyperClockTable::kMaxLoadFactor + + 0.999); +} + +} // namespace + +// An RAII wrapper for locking a chain of entries (flag bit on the head) +// so that there is only one thread allowed to remove entries from the +// chain, or to rewrite it by splitting for Grow. Without the lock, +// all lookups and insertions at the head can proceed wait-free. +// The class also provides functions for safely manipulating the head pointer +// while holding the lock--or wanting to should it become non-empty. +// +// The flag bits on the head are such that the head cannot be locked if it +// is an empty chain, so that a "blind" FetchOr will try to lock a non-empty +// chain but have no effect on an empty chain. When a potential rewrite +// operation see an empty head pointer, there is no need to lock as the +// operation is a no-op. However, there are some cases such as CAS-update +// where locking might be required after initially not being needed, if the +// operation is forced to revisit the head pointer. +class AutoHyperClockTable::ChainRewriteLock { + public: + using HandleImpl = AutoHyperClockTable::HandleImpl; + + // Acquire lock if head of h is not an end + explicit ChainRewriteLock(HandleImpl* h, RelaxedAtomic& yield_count) + : head_ptr_(&h->head_next_with_shift) { + Acquire(yield_count); + } + + // RAII wrap existing lock held (or end) + explicit ChainRewriteLock(HandleImpl* h, + RelaxedAtomic& /*yield_count*/, + uint64_t already_locked_or_end) + : head_ptr_(&h->head_next_with_shift) { + saved_head_ = already_locked_or_end; + // already locked or end + assert(saved_head_ & HandleImpl::kHeadLocked); + } + + ~ChainRewriteLock() { + if (!IsEnd()) { + // Release lock + uint64_t old = head_ptr_->FetchAnd(~HandleImpl::kHeadLocked); + (void)old; + assert((old & HandleImpl::kNextEndFlags) == HandleImpl::kHeadLocked); + } + } + + void Reset(HandleImpl* h, RelaxedAtomic& yield_count) { + this->~ChainRewriteLock(); + new (this) ChainRewriteLock(h, yield_count); + } + + // Expected current state, assuming no parallel updates. + uint64_t GetSavedHead() const { return saved_head_; } + + bool CasUpdate(uint64_t next_with_shift, + RelaxedAtomic& yield_count) { + uint64_t new_head = next_with_shift | HandleImpl::kHeadLocked; + uint64_t expected = GetSavedHead(); + bool success = head_ptr_->CasStrong(expected, new_head); + if (success) { + // Ensure IsEnd() is kept up-to-date, including for dtor + saved_head_ = new_head; + } else { + // Parallel update to head, such as Insert() + if (IsEnd()) { + // Didn't previously hold a lock + if (HandleImpl::IsEnd(expected)) { + // Still don't need to + saved_head_ = expected; + } else { + // Need to acquire lock before proceeding + Acquire(yield_count); + } + } else { + // Parallel update must preserve our lock + assert((expected & HandleImpl::kNextEndFlags) == + HandleImpl::kHeadLocked); + saved_head_ = expected; + } + } + return success; + } + + bool IsEnd() const { return HandleImpl::IsEnd(saved_head_); } + + private: + void Acquire(RelaxedAtomic& yield_count) { + for (;;) { + // Acquire removal lock on the chain + uint64_t old_head = head_ptr_->FetchOr(HandleImpl::kHeadLocked); + if ((old_head & HandleImpl::kNextEndFlags) != HandleImpl::kHeadLocked) { + // Either acquired the lock or lock not needed (end) + assert((old_head & HandleImpl::kNextEndFlags) == 0 || + (old_head & HandleImpl::kNextEndFlags) == + HandleImpl::kNextEndFlags); + + saved_head_ = old_head | HandleImpl::kHeadLocked; + break; + } + // NOTE: one of the few yield-wait loops, which is rare enough in practice + // for its performance to be insignificant. (E.g. using C++20 atomic + // wait/notify would likely be worse because of wasted notify costs.) + yield_count.FetchAddRelaxed(1); + std::this_thread::yield(); + } + } + + AcqRelAtomic* head_ptr_; + uint64_t saved_head_; +}; + +AutoHyperClockTable::AutoHyperClockTable( + size_t capacity, CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed, + const Opts& opts) + : BaseClockTable(metadata_charge_policy, allocator, eviction_callback, + hash_seed), + array_(MemMapping::AllocateLazyZeroed( + sizeof(HandleImpl) * CalcMaxUsableLength(capacity, + opts.min_avg_value_size, + metadata_charge_policy))), + length_info_(UsedLengthToLengthInfo(GetStartingLength(capacity))), + occupancy_limit_( + CalcOccupancyLimit(LengthInfoToUsedLength(length_info_.Load()))), + grow_frontier_(GetTableSize()), + clock_pointer_mask_( + BottomNBits(UINT64_MAX, LengthInfoToMinShift(length_info_.Load()))) { + if (metadata_charge_policy == + CacheMetadataChargePolicy::kFullChargeCacheMetadata) { + // NOTE: ignoring page boundaries for simplicity + usage_.FetchAddRelaxed(size_t{GetTableSize()} * sizeof(HandleImpl)); + } + + static_assert(sizeof(HandleImpl) == 64U, + "Expecting size / alignment with common cache line size"); + + // Populate head pointers + uint64_t length_info = length_info_.Load(); + int min_shift = LengthInfoToMinShift(length_info); + int max_shift = min_shift + 1; + size_t major = uint64_t{1} << min_shift; + size_t used_length = GetTableSize(); + + assert(major <= used_length); + assert(used_length <= major * 2); + + // Initialize the initial usable set of slots. This slightly odd iteration + // order makes it easier to get the correct shift amount on each head. + for (size_t i = 0; i < major; ++i) { +#ifndef NDEBUG + int shift; + size_t home; +#endif + if (major + i < used_length) { + array_[i].head_next_with_shift.StoreRelaxed( + MakeNextWithShiftEnd(i, max_shift)); + array_[major + i].head_next_with_shift.StoreRelaxed( + MakeNextWithShiftEnd(major + i, max_shift)); +#ifndef NDEBUG // Extra invariant checking + GetHomeIndexAndShift(length_info, i, &home, &shift); + assert(home == i); + assert(shift == max_shift); + GetHomeIndexAndShift(length_info, major + i, &home, &shift); + assert(home == major + i); + assert(shift == max_shift); +#endif + } else { + array_[i].head_next_with_shift.StoreRelaxed( + MakeNextWithShiftEnd(i, min_shift)); +#ifndef NDEBUG // Extra invariant checking + GetHomeIndexAndShift(length_info, i, &home, &shift); + assert(home == i); + assert(shift == min_shift); + GetHomeIndexAndShift(length_info, major + i, &home, &shift); + assert(home == i); + assert(shift == min_shift); +#endif + } + } +} + +AutoHyperClockTable::~AutoHyperClockTable() { + // As usual, destructor assumes there are no references or active operations + // on any slot/element in the table. + + // It's possible that there were not enough Insert() after final concurrent + // Grow to ensure length_info_ (published GetTableSize()) is fully up to + // date. Probe for first unused slot to ensure we see the whole structure. + size_t used_end = GetTableSize(); + while (used_end < array_.Count() && + array_[used_end].head_next_with_shift.LoadRelaxed() != + HandleImpl::kUnusedMarker) { + used_end++; + } +#ifndef NDEBUG + for (size_t i = used_end; i < array_.Count(); i++) { + assert(array_[i].head_next_with_shift.LoadRelaxed() == 0); + assert(array_[i].chain_next_with_shift.LoadRelaxed() == 0); + assert(array_[i].meta.LoadRelaxed() == 0); + } + std::vector was_populated(used_end); + std::vector was_pointed_to(used_end); +#endif + for (size_t i = 0; i < used_end; i++) { + HandleImpl& h = array_[i]; + switch (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) { + case ClockHandle::kStateEmpty: + // noop + break; + case ClockHandle::kStateInvisible: // rare but possible + case ClockHandle::kStateVisible: + assert(GetRefcount(h.meta.LoadRelaxed()) == 0); + h.FreeData(allocator_); +#ifndef NDEBUG // Extra invariant checking + usage_.FetchSubRelaxed(h.total_charge); + occupancy_.FetchSubRelaxed(1U); + was_populated[i] = true; + if (!HandleImpl::IsEnd(h.chain_next_with_shift.LoadRelaxed())) { + assert((h.chain_next_with_shift.LoadRelaxed() & + HandleImpl::kHeadLocked) == 0); + size_t next = + GetNextFromNextWithShift(h.chain_next_with_shift.LoadRelaxed()); + assert(!was_pointed_to[next]); + was_pointed_to[next] = true; + } +#endif + break; + // otherwise + default: + assert(false); + break; + } +#ifndef NDEBUG // Extra invariant checking + if (!HandleImpl::IsEnd(h.head_next_with_shift.LoadRelaxed())) { + size_t next = + GetNextFromNextWithShift(h.head_next_with_shift.LoadRelaxed()); + assert(!was_pointed_to[next]); + was_pointed_to[next] = true; + } +#endif + } +#ifndef NDEBUG // Extra invariant checking + // This check is not perfect, but should detect most reasonable cases + // of abandonned or floating entries, etc. (A floating cycle would not + // be reported as bad.) + for (size_t i = 0; i < used_end; i++) { + if (was_populated[i]) { + assert(was_pointed_to[i]); + } else { + assert(!was_pointed_to[i]); + } + } +#endif + + // Metadata charging only follows the published table size + assert(usage_.LoadRelaxed() == 0 || + usage_.LoadRelaxed() == GetTableSize() * sizeof(HandleImpl)); + assert(occupancy_.LoadRelaxed() == 0); +} + +size_t AutoHyperClockTable::GetTableSize() const { + return LengthInfoToUsedLength(length_info_.Load()); +} + +size_t AutoHyperClockTable::GetOccupancyLimit() const { + return occupancy_limit_.LoadRelaxed(); +} + +void AutoHyperClockTable::StartInsert(InsertState& state) { + state.saved_length_info = length_info_.Load(); +} + +// Because we have linked lists, bugs or even hardware errors can make it +// possible to create a cycle, which would lead to infinite loop. +// Furthermore, when we have retry cases in the code, we want to be sure +// these are not (and do not become) spin-wait loops. Given the assumption +// of quality hashing and the infeasibility of consistently recurring +// concurrent modifications to an entry or chain, we can safely bound the +// number of loop iterations in feasible operation, whether following chain +// pointers or retrying with some backtracking. A smaller limit is used for +// stress testing, to detect potential issues such as cycles or spin-waits, +// and a larger limit is used to break cycles should they occur in production. +#define CHECK_TOO_MANY_ITERATIONS(i) \ + { \ + assert(i < 768); \ + if (UNLIKELY(i >= 4096)) { \ + std::terminate(); \ + } \ + } + +bool AutoHyperClockTable::GrowIfNeeded(size_t new_occupancy, + InsertState& state) { + // new_occupancy has taken into account other threads that are also trying + // to insert, so as soon as we see sufficient *published* usable size, we + // can declare success even if we aren't the one that grows the table. + // However, there's an awkward state where other threads own growing the + // table to sufficient usable size, but the udpated size is not yet + // published. If we wait, then that likely slows the ramp-up cache + // performance. If we unblock ourselves by ensuring we grow by at least one + // slot, we could technically overshoot required size by number of parallel + // threads accessing block cache. On balance considering typical cases and + // the modest consequences of table being slightly too large, the latter + // seems preferable. + // + // So if the published occupancy limit is too small, we unblock ourselves + // by committing to growing the table by at least one slot. Also note that + // we might need to grow more than once to actually increase the occupancy + // limit (due to max load factor < 1.0) + + while (UNLIKELY(new_occupancy > occupancy_limit_.LoadRelaxed())) { + // At this point we commit the thread to growing unless we've reached the + // limit (returns false). + if (!Grow(state)) { + return false; + } + } + // Success (didn't need to grow, or did successfully) + return true; +} + +bool AutoHyperClockTable::Grow(InsertState& state) { + // Allocate the next grow slot + size_t grow_home = grow_frontier_.FetchAddRelaxed(1); + if (grow_home >= array_.Count()) { + // Can't grow any more. + // (Tested by unit test ClockCacheTest/Limits) + // Make sure we don't overflow grow_frontier_ by reaching here repeatedly + grow_frontier_.StoreRelaxed(array_.Count()); + return false; + } +#ifdef COERCE_CONTEXT_SWITCH + // This is useful in reproducing concurrency issues in Grow() + while (Random::GetTLSInstance()->OneIn(2)) { + std::this_thread::yield(); + } +#endif + // Basically, to implement https://en.wikipedia.org/wiki/Linear_hashing + // entries that belong in a new chain starting at grow_home will be + // split off from the chain starting at old_home, which is computed here. + int old_shift = FloorLog2(grow_home); + size_t old_home = BottomNBits(grow_home, old_shift); + assert(old_home + (size_t{1} << old_shift) == grow_home); + + // Wait here to ensure any Grow operations that would directly feed into + // this one are finished, though the full waiting actually completes in + // acquiring the rewrite lock for old_home in SplitForGrow. Here we ensure + // the expected shift amount has been reached, and there we ensure the + // chain rewrite lock has been released. + size_t old_old_home = BottomNBits(grow_home, old_shift - 1); + for (;;) { + uint64_t old_old_head = array_[old_old_home].head_next_with_shift.Load(); + if (GetShiftFromNextWithShift(old_old_head) >= old_shift) { + if ((old_old_head & HandleImpl::kNextEndFlags) != + HandleImpl::kHeadLocked) { + break; + } + } + // NOTE: one of the few yield-wait loops, which is rare enough in practice + // for its performance to be insignificant. + yield_count_.FetchAddRelaxed(1); + std::this_thread::yield(); + } + + // Do the dirty work of splitting the chain, including updating heads and + // chain nexts for new shift amounts. + SplitForGrow(grow_home, old_home, old_shift); + + // length_info_ can be updated any time after the new shift amount is + // published to both heads, potentially before the end of SplitForGrow. + // But we also can't update length_info_ until the previous Grow operation + // (with grow_home := this grow_home - 1) has published the new shift amount + // to both of its heads. However, we don't want to artificially wait here + // on that Grow that is otherwise irrelevant. + // + // We could have each Grow operation advance length_info_ here as far as it + // can without waiting, by checking for updated shift on the corresponding + // old home and also stopping at an empty head value for possible grow_home. + // However, this could increase CPU cache line sharing and in 1/64 cases + // bring in an extra page from our mmap. + // + // Instead, part of the strategy is delegated to DoInsert(): + // * Here we try to bring length_info_ up to date with this grow_home as + // much as we can without waiting. It will fall short if a previous Grow + // is still between reserving the grow slot and making the first big step + // to publish the new shift amount. + // * To avoid length_info_ being perpetually out-of-date (for a small number + // of heads) after our last Grow, we do the same when Insert has to "fall + // forward" due to length_info_ being out-of-date. + CatchUpLengthInfoNoWait(grow_home); + + // See usage in DoInsert() + state.likely_empty_slot = grow_home; + + // Success + return true; +} + +// See call in Grow() +void AutoHyperClockTable::CatchUpLengthInfoNoWait( + size_t known_usable_grow_home) { + uint64_t current_length_info = length_info_.Load(); + size_t published_usable_size = LengthInfoToUsedLength(current_length_info); + while (published_usable_size <= known_usable_grow_home) { + // For when published_usable_size was grow_home + size_t next_usable_size = published_usable_size + 1; + uint64_t next_length_info = UsedLengthToLengthInfo(next_usable_size); + + // known_usable_grow_home is known to be ready for Lookup/Insert with + // the new shift amount, but between that and published usable size, we + // need to check. + if (published_usable_size < known_usable_grow_home) { + int old_shift = FloorLog2(next_usable_size - 1); + size_t old_home = BottomNBits(published_usable_size, old_shift); + int shift = GetShiftFromNextWithShift( + array_[old_home].head_next_with_shift.Load()); + if (shift <= old_shift) { + // Not ready + break; + } + } + // CAS update length_info_. This only moves in one direction, so if CAS + // fails, someone else made progress like we are trying, and we can just + // pick up the new value and keep going as appropriate. + if (length_info_.CasStrong(current_length_info, next_length_info)) { + current_length_info = next_length_info; + // Update usage_ if metadata charge policy calls for it + if (metadata_charge_policy_ == + CacheMetadataChargePolicy::kFullChargeCacheMetadata) { + // NOTE: ignoring page boundaries for simplicity + usage_.FetchAddRelaxed(sizeof(HandleImpl)); + } + } + published_usable_size = LengthInfoToUsedLength(current_length_info); + } + + // After updating lengh_info_ we can update occupancy_limit_, + // allowing for later operations to update it before us. + // Note: there is no AcqRelAtomic max operation, so we have to use a CAS loop + size_t old_occupancy_limit = occupancy_limit_.LoadRelaxed(); + size_t new_occupancy_limit = CalcOccupancyLimit(published_usable_size); + while (old_occupancy_limit < new_occupancy_limit) { + if (occupancy_limit_.CasWeakRelaxed(old_occupancy_limit, + new_occupancy_limit)) { + break; + } + } +} + +void AutoHyperClockTable::SplitForGrow(size_t grow_home, size_t old_home, + int old_shift) { + int new_shift = old_shift + 1; + HandleImpl* const arr = array_.Get(); + + // We implement a somewhat complicated splitting algorithm to ensure that + // entries are always wait-free visible to Lookup, without Lookup needing + // to double-check length_info_ to ensure every potentially relevant + // existing entry is seen. This works step-by-step, carefully sharing + // unmigrated parts of the chain between the source chain and the new + // destination chain. This means that Lookup might see a partially migrated + // chain so has to take that into consideration when checking that it hasn't + // "jumped off" its intended chain (due to a parallel modification to an + // "under (de)construction" entry that was found on the chain but has + // been reassigned). + // + // We use a "rewrite lock" on the source and desination chains to exclude + // removals from those, and we have a prior waiting step that ensures any Grow + // operations feeding into this one have completed. But this process does have + // to gracefully handle concurrent insertions to the head of the source chain, + // and once marked ready, the destination chain. + // + // With those considerations, the migration starts with one "big step," + // potentially with retries to deal with insertions in parallel. Part of the + // big step is to mark the two chain heads as updated with the new shift + // amount, which redirects Lookups to the appropriate new chain. + // + // After that big step that updates the heads, the rewrite lock makes it + // relatively easy to deal with the rest of the migration. Big + // simplifications come from being able to read the hashed_key of each + // entry on the chain without needing to hold a read reference, and + // from never "jumping our to another chain." Concurrent insertions only + // happen at the chain head, which is outside of what is left to migrate. + // + // A series of smaller steps finishes splitting apart the existing chain into + // two distinct chains, followed by some steps to fully commit the result. + // + // Except for trivial cases in which all entries (or remaining entries) + // on the input chain go to one output chain, there is an important invariant + // after each step of migration, including after the initial "big step": + // For each output chain, the "zero chain" (new hash bit is zero) and the + // "one chain" (new hash bit is one) we have a "frontier" entry marking the + // boundary between what has been migrated and what has not. One of the + // frontiers is along the old chain after the other, and all entries between + // them are for the same target chain as the earlier frontier. Thus, the + // chains share linked list tails starting at the latter frontier. All + // pointers from the new head locations to the frontier entries are marked + // with the new shift amount, while all pointers after the frontiers use the + // old shift amount. + // + // And after each step there is a strengthening step to reach a stronger + // invariant: the frontier earlier in the original chain is advanced to be + // immediately before the other frontier. + // + // Consider this original input chain, + // + // OldHome -Old-> A0 -Old-> B0 -Old-> A1 -Old-> C0 -Old-> OldHome(End) + // GrowHome (empty) + // + // == BIG STEP == + // The initial big step finds the first entry that will be on the each + // output chain (in this case A0 and A1). We use brackets ([]) to mark them + // as our prospective frontiers. + // + // OldHome -Old-> [A0] -Old-> B0 -Old-> [A1] -Old-> C0 -Old-> OldHome(End) + // GrowHome (empty) + // + // Next we speculatively update grow_home head to point to the first entry for + // the one chain. This will not be used by Lookup until the head at old_home + // uses the new shift amount. + // + // OldHome -Old-> [A0] -Old-> B0 -Old-> [A1] -Old-> C0 -Old-> OldHome(End) + // GrowHome --------------New------------/ + // + // Observe that if Lookup were to use the new head at GrowHome, it would be + // able to find all relevant entries. Finishing the initial big step + // requires a CAS (compare_exchange) of the OldHome head because there + // might have been parallel insertions there, in which case we roll back + // and try again. (We might need to point GrowHome head differently.) + // + // OldHome -New-> [A0] -Old-> B0 -Old-> [A1] -Old-> C0 -Old-> OldHome(End) + // GrowHome --------------New------------/ + // + // Upgrading the OldHome head pointer with the new shift amount, with a + // compare_exchange, completes the initial big step, with [A0] as zero + // chain frontier and [A1] as one chain frontier. Links before the frontiers + // use the new shift amount and links after use the old shift amount. + // == END BIG STEP== + // == STRENGTHENING == + // Zero chain frontier is advanced to [B0] (immediately before other + // frontier) by updating pointers with new shift amounts. + // + // OldHome -New-> A0 -New-> [B0] -Old-> [A1] -Old-> C0 -Old-> OldHome(End) + // GrowHome -------------New-----------/ + // + // == END STRENGTHENING == + // == SMALL STEP #1 == + // From the strong invariant state, we need to find the next entry for + // the new chain with the earlier frontier. In this case, we need to find + // the next entry for the zero chain that comes after [B0], which in this + // case is C0. This will be our next zero chain frontier, at least under + // the weak invariant. To get there, we simply update the link between + // the current two frontiers to skip over the entries irreleveant to the + // ealier frontier chain. In this case, the zero chain skips over A1. As a + // result, he other chain is now the "earlier." + // + // OldHome -New-> A0 -New-> B0 -New-> [C0] -Old-> OldHome(End) + // GrowHome -New-> [A1] ------Old-----/ + // + // == END SMALL STEP #1 == + // + // Repeating the cycle and end handling is not as interesting. + + // Acquire rewrite lock on zero chain (if it's non-empty) + ChainRewriteLock zero_head_lock(&arr[old_home], yield_count_); + + // Used for locking the one chain below + uint64_t saved_one_head; + // One head has not been written to + assert(arr[grow_home].head_next_with_shift.Load() == 0); + + // old_home will also the head of the new "zero chain" -- all entries in the + // "from" chain whose next hash bit is 0. grow_home will be head of the new + // "one chain". + + // For these, SIZE_MAX is like nullptr (unknown) + size_t zero_chain_frontier = SIZE_MAX; + size_t one_chain_frontier = SIZE_MAX; + size_t cur = SIZE_MAX; + + // Set to 0 (zero chain frontier earlier), 1 (one chain), or -1 (unknown) + int chain_frontier_first = -1; + + // Might need to retry initial update of heads + for (int i = 0;; ++i) { + CHECK_TOO_MANY_ITERATIONS(i); + assert(zero_chain_frontier == SIZE_MAX); + assert(one_chain_frontier == SIZE_MAX); + assert(cur == SIZE_MAX); + assert(chain_frontier_first == -1); + + uint64_t next_with_shift = zero_head_lock.GetSavedHead(); + + // Find a single representative for each target chain, or scan the whole + // chain if some target chain has no representative. + for (;; ++i) { + CHECK_TOO_MANY_ITERATIONS(i); + + // Loop invariants + assert((chain_frontier_first < 0) == (zero_chain_frontier == SIZE_MAX && + one_chain_frontier == SIZE_MAX)); + assert((cur == SIZE_MAX) == (zero_chain_frontier == SIZE_MAX && + one_chain_frontier == SIZE_MAX)); + + assert(GetShiftFromNextWithShift(next_with_shift) == old_shift); + + // Check for end of original chain + if (HandleImpl::IsEnd(next_with_shift)) { + cur = SIZE_MAX; + break; + } + + // next_with_shift is not End + cur = GetNextFromNextWithShift(next_with_shift); + + if (BottomNBits(arr[cur].hashed_key[1], new_shift) == old_home) { + // Entry for zero chain + if (zero_chain_frontier == SIZE_MAX) { + zero_chain_frontier = cur; + if (one_chain_frontier != SIZE_MAX) { + // Ready to update heads + break; + } + // Nothing yet for one chain + chain_frontier_first = 0; + } + } else { + assert(BottomNBits(arr[cur].hashed_key[1], new_shift) == grow_home); + // Entry for one chain + if (one_chain_frontier == SIZE_MAX) { + one_chain_frontier = cur; + if (zero_chain_frontier != SIZE_MAX) { + // Ready to update heads + break; + } + // Nothing yet for zero chain + chain_frontier_first = 1; + } + } + + next_with_shift = arr[cur].chain_next_with_shift.Load(); + } + + // Try to update heads for initial migration info + // We only reached the end of the migrate-from chain already if one of the + // target chains will be empty. + assert((cur == SIZE_MAX) == + (zero_chain_frontier == SIZE_MAX || one_chain_frontier == SIZE_MAX)); + assert((chain_frontier_first < 0) == + (zero_chain_frontier == SIZE_MAX && one_chain_frontier == SIZE_MAX)); + + // Always update one chain's head first (safe), and mark it as locked + saved_one_head = HandleImpl::kHeadLocked | + (one_chain_frontier != SIZE_MAX + ? MakeNextWithShift(one_chain_frontier, new_shift) + : MakeNextWithShiftEnd(grow_home, new_shift)); + arr[grow_home].head_next_with_shift.Store(saved_one_head); + + // Make sure length_info_ hasn't been updated too early, as we're about + // to make the change that makes it safe to update (e.g. in DoInsert()) + assert(LengthInfoToUsedLength(length_info_.Load()) <= grow_home); + + // Try to set zero's head. + if (zero_head_lock.CasUpdate( + zero_chain_frontier != SIZE_MAX + ? MakeNextWithShift(zero_chain_frontier, new_shift) + : MakeNextWithShiftEnd(old_home, new_shift), + yield_count_)) { + // Both heads successfully updated to new shift + break; + } else { + // Concurrent insertion. This should not happen too many times. + CHECK_TOO_MANY_ITERATIONS(i); + // The easiest solution is to restart. + zero_chain_frontier = SIZE_MAX; + one_chain_frontier = SIZE_MAX; + cur = SIZE_MAX; + chain_frontier_first = -1; + continue; + } + } + + // Create an RAII wrapper for the one chain rewrite lock we are already + // holding (if was not end) and is now "published" after successful CAS on + // zero chain head. + ChainRewriteLock one_head_lock(&arr[grow_home], yield_count_, saved_one_head); + + // Except for trivial cases, we have something like + // AHome -New-> [A0] -Old-> [B0] -Old-> [C0] \ | + // BHome --------------------New------------> [A1] -Old-> ... + // And we need to upgrade as much as we can on the "first" chain + // (the one eventually pointing to the other's frontier). This will + // also finish off any case in which one of the target chains will be empty. + if (chain_frontier_first >= 0) { + size_t& first_frontier = chain_frontier_first == 0 + ? /*&*/ zero_chain_frontier + : /*&*/ one_chain_frontier; + size_t& other_frontier = chain_frontier_first != 0 + ? /*&*/ zero_chain_frontier + : /*&*/ one_chain_frontier; + uint64_t stop_before_or_new_tail = + other_frontier != SIZE_MAX + ? /*stop before*/ MakeNextWithShift(other_frontier, old_shift) + : /*new tail*/ MakeNextWithShiftEnd( + chain_frontier_first == 0 ? old_home : grow_home, new_shift); + UpgradeShiftsOnRange(arr, first_frontier, stop_before_or_new_tail, + old_shift, new_shift); + } + + if (zero_chain_frontier == SIZE_MAX) { + // Already finished migrating + assert(one_chain_frontier == SIZE_MAX); + assert(cur == SIZE_MAX); + } else { + // Still need to migrate between two target chains + for (int i = 0;; ++i) { + CHECK_TOO_MANY_ITERATIONS(i); + // Overall loop invariants + assert(zero_chain_frontier != SIZE_MAX); + assert(one_chain_frontier != SIZE_MAX); + assert(cur != SIZE_MAX); + assert(chain_frontier_first >= 0); + size_t& first_frontier = chain_frontier_first == 0 + ? /*&*/ zero_chain_frontier + : /*&*/ one_chain_frontier; + size_t& other_frontier = chain_frontier_first != 0 + ? /*&*/ zero_chain_frontier + : /*&*/ one_chain_frontier; + assert(cur != first_frontier); + assert(GetNextFromNextWithShift( + arr[first_frontier].chain_next_with_shift.Load()) == + other_frontier); + + uint64_t next_with_shift = arr[cur].chain_next_with_shift.Load(); + + // Check for end of original chain + if (HandleImpl::IsEnd(next_with_shift)) { + // Can set upgraded tail on first chain + uint64_t first_new_tail = MakeNextWithShiftEnd( + chain_frontier_first == 0 ? old_home : grow_home, new_shift); + arr[first_frontier].chain_next_with_shift.Store(first_new_tail); + // And upgrade remainder of other chain + uint64_t other_new_tail = MakeNextWithShiftEnd( + chain_frontier_first != 0 ? old_home : grow_home, new_shift); + UpgradeShiftsOnRange(arr, other_frontier, other_new_tail, old_shift, + new_shift); + assert(other_frontier == SIZE_MAX); // Finished + break; + } + + // next_with_shift is not End + cur = GetNextFromNextWithShift(next_with_shift); + + int target_chain; + if (BottomNBits(arr[cur].hashed_key[1], new_shift) == old_home) { + // Entry for zero chain + target_chain = 0; + } else { + assert(BottomNBits(arr[cur].hashed_key[1], new_shift) == grow_home); + // Entry for one chain + target_chain = 1; + } + if (target_chain == chain_frontier_first) { + // Found next entry to skip to on the first chain + uint64_t skip_to = MakeNextWithShift(cur, new_shift); + arr[first_frontier].chain_next_with_shift.Store(skip_to); + first_frontier = cur; + // Upgrade other chain up to entry before that one + UpgradeShiftsOnRange(arr, other_frontier, next_with_shift, old_shift, + new_shift); + // Swap which is marked as first + chain_frontier_first = 1 - chain_frontier_first; + } else { + // Nothing to do yet, as we need to keep old generation pointers in + // place for lookups + } + } + } +} + +// Variant of PurgeImplLocked: Removes all "under (de) construction" entries +// from a chain where already holding a rewrite lock +using PurgeLockedOpData = void; +// Variant of PurgeImplLocked: Clock-updates all entries in a chain, in +// addition to functionality of PurgeLocked, where already holding a rewrite +// lock. (Caller finalizes eviction on entries added to the autovector, in part +// so that we don't hold the rewrite lock while doing potentially expensive +// callback and allocator free.) +using ClockUpdateChainLockedOpData = + autovector; + +template +void AutoHyperClockTable::PurgeImplLocked(OpData* op_data, + ChainRewriteLock& rewrite_lock, + size_t home, + BaseClockTable::EvictionData* data) { + constexpr bool kIsPurge = std::is_same_v; + constexpr bool kIsClockUpdateChain = + std::is_same_v; + + // Exactly one op specified + static_assert(kIsPurge + kIsClockUpdateChain == 1); + + HandleImpl* const arr = array_.Get(); + + uint64_t next_with_shift = rewrite_lock.GetSavedHead(); + assert(!HandleImpl::IsEnd(next_with_shift)); + int home_shift = GetShiftFromNextWithShift(next_with_shift); + (void)home; + (void)home_shift; + size_t next = GetNextFromNextWithShift(next_with_shift); + assert(next < array_.Count()); + HandleImpl* h = &arr[next]; + HandleImpl* prev_to_keep = nullptr; +#ifndef NDEBUG + uint64_t prev_to_keep_next_with_shift = 0; +#endif + // Whether there are entries between h and prev_to_keep that should be + // purged from the chain. + bool pending_purge = false; + + // Walk the chain, and stitch together any entries that are still + // "shareable," possibly after clock update. prev_to_keep tells us where + // the last "stitch back to" location is (nullptr => head). + for (size_t i = 0;; ++i) { + CHECK_TOO_MANY_ITERATIONS(i); + + bool purgeable = false; + // In last iteration, h will be nullptr, to stitch together the tail of + // the chain. + if (h) { + // NOTE: holding a rewrite lock on the chain prevents any "under + // (de)construction" entries in the chain from being marked empty, which + // allows us to access the hashed_keys without holding a read ref. + assert(home == BottomNBits(h->hashed_key[1], home_shift)); + if constexpr (kIsClockUpdateChain) { + // Clock update and/or check for purgeable (under (de)construction) + if (ClockUpdate(*h, data, &purgeable)) { + // Remember for finishing eviction + op_data->push_back(h); + // Entries for eviction become purgeable + purgeable = true; + assert((h->meta.Load() >> ClockHandle::kStateShift) == + ClockHandle::kStateConstruction); + } + } else { + (void)op_data; + (void)data; + purgeable = ((h->meta.Load() >> ClockHandle::kStateShift) & + ClockHandle::kStateShareableBit) == 0; + } + } + + if (purgeable) { + assert((h->meta.Load() >> ClockHandle::kStateShift) == + ClockHandle::kStateConstruction); + pending_purge = true; + } else if (pending_purge) { + if (prev_to_keep) { + // Update chain next to skip purgeable entries + assert(prev_to_keep->chain_next_with_shift.Load() == + prev_to_keep_next_with_shift); + prev_to_keep->chain_next_with_shift.Store(next_with_shift); + } else if (rewrite_lock.CasUpdate(next_with_shift, yield_count_)) { + // Managed to update head without any parallel insertions + } else { + // Parallel insertion must have interfered. Need to do a purge + // from updated head to here. Since we have no prev_to_keep, there's + // no risk of duplicate clock updates to entries. Any entries already + // updated must have been evicted (purgeable) and it's OK to clock + // update any new entries just inserted in parallel. + // Can simply restart (GetSavedHead() already updated from CAS failure). + next_with_shift = rewrite_lock.GetSavedHead(); + assert(!HandleImpl::IsEnd(next_with_shift)); + next = GetNextFromNextWithShift(next_with_shift); + assert(next < array_.Count()); + h = &arr[next]; + pending_purge = false; + assert(prev_to_keep == nullptr); + assert(GetShiftFromNextWithShift(next_with_shift) == home_shift); + continue; + } + pending_purge = false; + prev_to_keep = h; + } else { + prev_to_keep = h; + } + + if (h == nullptr) { + // Reached end of the chain + return; + } + + // Read chain pointer + next_with_shift = h->chain_next_with_shift.Load(); +#ifndef NDEBUG + if (prev_to_keep == h) { + prev_to_keep_next_with_shift = next_with_shift; + } +#endif + + assert(GetShiftFromNextWithShift(next_with_shift) == home_shift); + + // Check for end marker + if (HandleImpl::IsEnd(next_with_shift)) { + h = nullptr; + } else { + next = GetNextFromNextWithShift(next_with_shift); + assert(next < array_.Count()); + h = &arr[next]; + assert(h != prev_to_keep); + } + } +} + +// Variant of PurgeImpl: Removes all "under (de) construction" entries in a +// chain, such that any entry with the given key must have been purged. +using PurgeOpData = const UniqueId64x2; +// Variant of PurgeImpl: Clock-updates all entries in a chain, in addition to +// purging as appropriate. (Caller finalizes eviction on entries added to the +// autovector, in part so that we don't hold the rewrite lock while doing +// potentially expensive callback and allocator free.) +using ClockUpdateChainOpData = ClockUpdateChainLockedOpData; + +template +void AutoHyperClockTable::PurgeImpl(OpData* op_data, size_t home, + BaseClockTable::EvictionData* data) { + // Early efforts to make AutoHCC fully wait-free ran into too many problems + // that needed obscure and potentially inefficient work-arounds to have a + // chance at working. + // + // The implementation settled on "essentially wait-free" which can be + // achieved by locking at the level of each probing chain and only for + // operations that might remove entries from the chain. Because parallel + // clock updates and Grow operations are ordered, contention is very rare. + // However, parallel insertions at any chain head have to be accommodated + // to keep them wait-free. + // + // This function implements Purge and ClockUpdateChain functions (see above + // OpData type definitions) as part of higher-level operations. This function + // ensures the correct chain is (eventually) covered and handles rewrite + // locking the chain. PurgeImplLocked has lower level details. + // + // In general, these operations and Grow are kept simpler by allowing eager + // purging of under (de-)construction entries. For example, an Erase + // operation might find that another thread has purged the entry from the + // chain by the time its own purge operation acquires the rewrite lock and + // proceeds. This is OK, and potentially reduces the number of lock/unlock + // cycles because empty chains are not rewrite-lockable. + + constexpr bool kIsPurge = std::is_same_v; + constexpr bool kIsClockUpdateChain = + std::is_same_v; + + // Exactly one op specified + static_assert(kIsPurge + kIsClockUpdateChain == 1); + + int home_shift = 0; + if constexpr (kIsPurge) { + // Purge callers leave home unspecified, to be determined from key + assert(home == SIZE_MAX); + GetHomeIndexAndShift(length_info_.Load(), (*op_data)[1], &home, + &home_shift); + assert(home_shift > 0); + } else { + assert(kIsClockUpdateChain); + // Evict callers must specify home + assert(home < SIZE_MAX); + } + + HandleImpl* const arr = array_.Get(); + + // Acquire the RAII rewrite lock (if not an empty chain) + ChainRewriteLock rewrite_lock(&arr[home], yield_count_); + + if constexpr (kIsPurge) { + // Ensure we are at the correct home for the shift in effect for the + // chain head. + for (;;) { + int shift = GetShiftFromNextWithShift(rewrite_lock.GetSavedHead()); + + if (shift > home_shift) { + // Found a newer shift at candidate head, which must apply to us. + // Newer shift might not yet be reflected in length_info_ (an atomicity + // gap in Grow), so operate as if it is. Note that other insertions + // could happen using this shift before length_info_ is updated, and + // it's possible (though unlikely) that multiple generations of Grow + // have occurred. If shift is more than one generation ahead of + // home_shift, it's possible that not all descendent homes have + // reached the `shift` generation. Thus, we need to advance only one + // shift at a time looking for a home+head with a matching shift + // amount. + home_shift++; + home = GetHomeIndex((*op_data)[1], home_shift); + rewrite_lock.Reset(&arr[home], yield_count_); + continue; + } else { + assert(shift == home_shift); + } + break; + } + } + + // If the chain is empty, nothing to do + if (!rewrite_lock.IsEnd()) { + if constexpr (kIsPurge) { + PurgeLockedOpData* locked_op_data{}; + PurgeImplLocked(locked_op_data, rewrite_lock, home, data); + } else { + PurgeImplLocked(op_data, rewrite_lock, home, data); + } + } +} + +AutoHyperClockTable::HandleImpl* AutoHyperClockTable::DoInsert( + const ClockHandleBasicData& proto, uint64_t initial_countdown, + bool take_ref, InsertState& state) { + size_t home; + int orig_home_shift; + GetHomeIndexAndShift(state.saved_length_info, proto.hashed_key[1], &home, + &orig_home_shift); + HandleImpl* const arr = array_.Get(); + + // We could go searching through the chain for any duplicate, but that's + // not typically helpful, except for the REDUNDANT block cache stats. + // (Inferior duplicates will age out with eviction.) However, we do skip + // insertion if the home slot (or some other we happen to probe) already + // has a match (already_matches below). This helps to keep better locality + // when we can. + // + // And we can do that as part of searching for an available slot to + // insert the new entry, because our preferred location and first slot + // checked will be the home slot. + // + // As the table initially grows to size, few entries will be in the same + // cache line as the chain head. However, churn in the cache relatively + // quickly improves the proportion of entries sharing that cache line with + // the chain head. Data: + // + // Initial population only: (cache_bench with -ops_per_thread=1) + // Entries at home count: 29,202 (out of 129,170 entries in 94,411 chains) + // Approximate average cache lines read to find an existing entry: + // 129.2 / 94.4 [without the heads] + // + (94.4 - 29.2) / 94.4 [the heads not included with entries] + // = 2.06 cache lines + // + // After 10 million ops: (-threads=10 -ops_per_thread=100000) + // Entries at home count: 67,556 (out of 129,359 entries in 94,756 chains) + // That's a majority of entries and more than 2/3rds of chains. + // Approximate average cache lines read to find an existing entry: + // = 1.65 cache lines + + // Even if we aren't saving a ref to this entry (take_ref == false), we need + // to keep a reference while we are inserting the entry into a chain, so that + // it is not erased by another thread while trying to insert it on the chain. + constexpr bool initial_take_ref = true; + + size_t used_length = LengthInfoToUsedLength(state.saved_length_info); + assert(home < used_length); + + size_t idx = home; + bool already_matches = false; + bool already_matches_ignore = false; + if (TryInsert(proto, arr[idx], initial_countdown, initial_take_ref, + &already_matches)) { + assert(idx == home); + } else if (already_matches) { + return nullptr; + // Here we try to populate newly-opened slots in the table, but not + // when we can add something to its home slot. This makes the structure + // more performant more quickly on (initial) growth. We ignore "already + // matches" in this case because it is unlikely and difficult to + // incorporate logic for here cleanly and efficiently. + } else if (UNLIKELY(state.likely_empty_slot > 0) && + TryInsert(proto, arr[state.likely_empty_slot], initial_countdown, + initial_take_ref, &already_matches_ignore)) { + idx = state.likely_empty_slot; + } else { + // We need to search for an available slot outside of the home. + // Linear hashing provides nice resizing but does typically mean + // that some heads (home locations) have (in expectation) twice as + // many entries mapped to them as other heads. For example if the + // usable length is 80, then heads 16-63 are (in expectation) twice + // as loaded as heads 0-15 and 64-79, which are using another hash bit. + // + // This means that if we just use linear probing (by a small constant) + // to find an available slot, part of the structure could easily fill up + // and resort to linear time operations even when the overall load factor + // is only modestly high, like 70%. Even though each slot has its own CPU + // cache line, there appears to be a small locality benefit (e.g. TLB and + // paging) to iterating one by one, as long as we don't afoul of the + // linear hashing imbalance. + // + // In a traditional non-concurrent structure, we could keep a "free list" + // to ensure immediate access to an available slot, but maintaining such + // a structure could require more cross-thread coordination to ensure + // all entries are eventually available to all threads. + // + // The way we solve this problem is to use unit-increment linear probing + // with a small bound, and then fall back on big jumps to have a good + // chance of finding a slot in an under-populated region quickly if that + // doesn't work. + size_t i = 0; + constexpr size_t kMaxLinearProbe = 4; + for (; i < kMaxLinearProbe; i++) { + idx++; + if (idx >= used_length) { + idx -= used_length; + } + if (TryInsert(proto, arr[idx], initial_countdown, initial_take_ref, + &already_matches)) { + break; + } + if (already_matches) { + return nullptr; + } + } + if (i == kMaxLinearProbe) { + // Keep searching, but change to a search method that should quickly + // find any under-populated region. Switching to an increment based + // on the golden ratio helps with that, but we also inject some minor + // variation (less than 2%, 1 in 2^6) to avoid clustering effects on + // this larger increment (if it were a fixed value in steady state + // operation). Here we are primarily using upper bits of hashed_key[1] + // while home is based on lowest bits. + uint64_t incr_ratio = 0x9E3779B185EBCA87U + (proto.hashed_key[1] >> 6); + size_t incr = FastRange64(incr_ratio, used_length); + assert(incr > 0); + size_t start = idx; + for (;; i++) { + idx += incr; + if (idx >= used_length) { + // Wrap around (faster than %) + idx -= used_length; + } + if (idx == start) { + // We have just completed a cycle that might not have covered all + // slots. (incr and used_length could have common factors.) + // Increment for the next cycle, which eventually ensures complete + // iteration over the set of slots before repeating. + idx++; + if (idx >= used_length) { + idx -= used_length; + } + start++; + if (start >= used_length) { + start -= used_length; + } + if (i >= used_length) { + used_length = LengthInfoToUsedLength(length_info_.Load()); + if (i >= used_length * 2) { + // Cycling back should not happen unless there is enough random + // churn in parallel that we happen to hit each slot at a time + // that it's occupied, which is really only feasible for small + // structures, though with linear probing to find empty slots, + // "small" here might be larger than for double hashing. + assert(used_length <= 256); + // Fall back on standalone insert in case something goes awry to + // cause this + return nullptr; + } + } + } + if (TryInsert(proto, arr[idx], initial_countdown, initial_take_ref, + &already_matches)) { + break; + } + if (already_matches) { + return nullptr; + } + } + } + } + + // Now insert into chain using head pointer + uint64_t next_with_shift; + int home_shift = orig_home_shift; + + // Might need to retry + for (int i = 0;; ++i) { + CHECK_TOO_MANY_ITERATIONS(i); + next_with_shift = arr[home].head_next_with_shift.Load(); + int shift = GetShiftFromNextWithShift(next_with_shift); + + if (UNLIKELY(shift != home_shift)) { + // NOTE: shift increases with table growth + if (shift > home_shift) { + // Must be grow in progress or completed since reading length_info. + // Pull out one more hash bit. (See Lookup() for why we can't + // safely jump to the shift that was read.) + home_shift++; + uint64_t hash_bit_mask = uint64_t{1} << (home_shift - 1); + assert((home & hash_bit_mask) == 0); + // BEGIN leftover updates to length_info_ for Grow() + size_t grow_home = home + hash_bit_mask; + assert(arr[grow_home].head_next_with_shift.Load() != + HandleImpl::kUnusedMarker); + CatchUpLengthInfoNoWait(grow_home); + // END leftover updates to length_info_ for Grow() + home += proto.hashed_key[1] & hash_bit_mask; + continue; + } else { + // Should not happen because length_info_ is only updated after both + // old and new home heads are marked with new shift + assert(false); + } + } + + // Values to update to + uint64_t head_next_with_shift = MakeNextWithShift(idx, home_shift); + uint64_t chain_next_with_shift = next_with_shift; + + // Preserve the locked state in head, without propagating to chain next + // where it is meaningless (and not allowed) + if (UNLIKELY((next_with_shift & HandleImpl::kNextEndFlags) == + HandleImpl::kHeadLocked)) { + head_next_with_shift |= HandleImpl::kHeadLocked; + chain_next_with_shift &= ~HandleImpl::kHeadLocked; + } + + arr[idx].chain_next_with_shift.Store(chain_next_with_shift); + if (arr[home].head_next_with_shift.CasWeak(next_with_shift, + head_next_with_shift)) { + // Success + if (!take_ref) { + Unref(arr[idx]); + } + return arr + idx; + } + } +} + +AutoHyperClockTable::HandleImpl* AutoHyperClockTable::Lookup( + const UniqueId64x2& hashed_key) { + // Lookups are wait-free with low occurrence of retries, back-tracking, + // and fallback. We do not have the benefit of holding a rewrite lock on + // the chain so must be prepared for many kinds of mayhem, most notably + // "falling off our chain" where a slot that Lookup has identified but + // has not read-referenced is removed from one chain and inserted into + // another. The full algorithm uses the following mitigation strategies to + // ensure every relevant entry inserted before this Lookup, and not yet + // evicted, is seen by Lookup, without excessive backtracking etc.: + // * Keep a known good read ref in the chain for "island hopping." When + // we observe that a concurrent write takes us off to another chain, we + // only need to fall back to our last known good read ref (most recent + // entry on the chain that is not "under construction," which is a transient + // state). We don't want to compound the CPU toil of a long chain with + // operations that might need to retry from scratch, with probability + // in proportion to chain length. + // * Only detect a chain is potentially incomplete because of a Grow in + // progress by looking at shift in the next pointer tags (rather than + // re-checking length_info_). + // * SplitForGrow, Insert, and PurgeImplLocked ensure that there are no + // transient states that might cause this full Lookup algorithm to skip over + // live entries. + + // Reading length_info_ is not strictly required for Lookup, if we were + // to increment shift sizes until we see a shift size match on the + // relevant head pointer. Thus, reading with relaxed memory order gives + // us a safe and almost always up-to-date jump into finding the correct + // home and head. + size_t home; + int home_shift; + GetHomeIndexAndShift(length_info_.LoadRelaxed(), hashed_key[1], &home, + &home_shift); + assert(home_shift > 0); + + // The full Lookup algorithm however is not great for hot path efficiency, + // because of the extra careful tracking described above. Overwhelmingly, + // we can find what we're looking for with a naive linked list traversal + // of the chain. Even if we "fall off our chain" to another, we don't + // violate memory safety. We just won't match the key we're looking for. + // And we would eventually reach an end state, possibly even experiencing a + // cycle as an entry is freed and reused during our traversal (though at + // any point in time the structure doesn't have cycles). + // + // So for hot path efficiency, we start with a naive Lookup attempt, and + // then fall back on full Lookup if we don't find the correct entry. To + // cap how much we invest into the naive Lookup, we simply cap the traversal + // length before falling back. Also, when we do fall back on full Lookup, + // we aren't paying much penalty by starting over. Much or most of the cost + // of Lookup is memory latency in following the chain pointers, and the + // naive Lookup has warmed the CPU cache for these entries, using as tight + // of a loop as possible. + + HandleImpl* const arr = array_.Get(); + uint64_t next_with_shift = arr[home].head_next_with_shift.LoadRelaxed(); + for (size_t i = 0; !HandleImpl::IsEnd(next_with_shift) && i < 10; ++i) { + HandleImpl* h = &arr[GetNextFromNextWithShift(next_with_shift)]; + // Attempt cheap key match without acquiring a read ref. This could give a + // false positive, which is re-checked after acquiring read ref, or false + // negative, which is re-checked in the full Lookup. Also, this is a + // technical UB data race according to TSAN, but we don't need to read + // a "correct" value here for correct overall behavior. +#ifdef __SANITIZE_THREAD__ + bool probably_equal = Random::GetTLSInstance()->OneIn(2); +#else + bool probably_equal = h->hashed_key == hashed_key; +#endif + if (probably_equal) { + // Increment acquire counter for definitive check + uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement); + // Check if it's a referencable (sharable) entry + if (LIKELY(old_meta & (uint64_t{ClockHandle::kStateShareableBit} + << ClockHandle::kStateShift))) { + assert(GetRefcount(old_meta + ClockHandle::kAcquireIncrement) > 0); + if (LIKELY(h->hashed_key == hashed_key) && + LIKELY(old_meta & (uint64_t{ClockHandle::kStateVisibleBit} + << ClockHandle::kStateShift))) { + return h; + } else { + Unref(*h); + } + } else { + // For non-sharable states, incrementing the acquire counter has no + // effect so we don't need to undo it. Furthermore, we cannot safely + // undo it because we did not acquire a read reference to lock the entry + // in a Shareable state. + } + } + + next_with_shift = h->chain_next_with_shift.LoadRelaxed(); + } + + // If we get here, falling back on full Lookup algorithm. + HandleImpl* h = nullptr; + HandleImpl* read_ref_on_chain = nullptr; + + for (size_t i = 0;; ++i) { + CHECK_TOO_MANY_ITERATIONS(i); + // Read head or chain pointer + next_with_shift = h ? h->chain_next_with_shift.Load() + : arr[home].head_next_with_shift.Load(); + int shift = GetShiftFromNextWithShift(next_with_shift); + + // Make sure it's usable + size_t effective_home = home; + if (UNLIKELY(shift != home_shift)) { + // We have potentially gone awry somehow, but it's possible we're just + // hitting old data that is not yet completed Grow. + // NOTE: shift bits goes up with table growth. + if (shift < home_shift) { + // To avoid waiting on Grow in progress, an old shift amount needs + // to be processed as if we were still using it and (potentially + // different or the same) the old home. + // We can assert it's not too old, because each generation of Grow + // waits on its ancestor in the previous generation. + assert(shift + 1 == home_shift); + effective_home = GetHomeIndex(home, shift); + } else if (h == read_ref_on_chain) { + assert(shift > home_shift); + // At head or coming from an entry on our chain where we're holding + // a read reference. Thus, we know the newer shift applies to us. + // Newer shift might not yet be reflected in length_info_ (an atomicity + // gap in Grow), so operate as if it is. Note that other insertions + // could happen using this shift before length_info_ is updated, and + // it's possible (though unlikely) that multiple generations of Grow + // have occurred. If shift is more than one generation ahead of + // home_shift, it's possible that not all descendent homes have + // reached the `shift` generation. Thus, we need to advance only one + // shift at a time looking for a home+head with a matching shift + // amount. + home_shift++; + // Update home in case it has changed + home = GetHomeIndex(hashed_key[1], home_shift); + // This should be rare enough occurrence that it's simplest just + // to restart (TODO: improve in some cases?) + h = nullptr; + if (read_ref_on_chain) { + Unref(*read_ref_on_chain); + read_ref_on_chain = nullptr; + } + // Didn't make progress & retry + continue; + } else { + assert(shift > home_shift); + assert(h != nullptr); + // An "under (de)construction" entry has a new shift amount, which + // means we have either gotten off our chain or our home shift is out + // of date. If we revert back to saved ref, we will get updated info. + h = read_ref_on_chain; + // Didn't make progress & retry + continue; + } + } + + // Check for end marker + if (HandleImpl::IsEnd(next_with_shift)) { + // To ensure we didn't miss anything in the chain, the end marker must + // point back to the correct home. + if (LIKELY(GetNextFromNextWithShift(next_with_shift) == effective_home)) { + // Complete, clean iteration of the chain, not found. + // Clean up. + if (read_ref_on_chain) { + Unref(*read_ref_on_chain); + } + return nullptr; + } else { + // Something went awry. Revert back to a safe point (if we have it) + h = read_ref_on_chain; + // Didn't make progress & retry + continue; + } + } + + // Follow the next and check for full key match, home match, or neither + h = &arr[GetNextFromNextWithShift(next_with_shift)]; + bool full_match_or_unknown = false; + if (MatchAndRef(&hashed_key, *h, shift, effective_home, + &full_match_or_unknown)) { + // Got a read ref on next (h). + // + // There is a very small chance that between getting the next pointer + // (now h) and doing MatchAndRef on it, another thread erased/evicted it + // reinserted it into the same chain, causing us to cycle back in the + // same chain and potentially see some entries again if we keep walking. + // Newly-inserted entries are inserted before older ones, so we are at + // least guaranteed not to miss anything. Here in Lookup, it's just a + // transient, slight hiccup in performance. + + if (full_match_or_unknown) { + // Full match. + // Release old read ref on chain if applicable + if (read_ref_on_chain) { + // Pretend we never took the reference. + Unref(*read_ref_on_chain); + } + // Update the hit bit + if (eviction_callback_) { + h->meta.FetchOrRelaxed(uint64_t{1} << ClockHandle::kHitBitShift); + } + // All done. + return h; + } else if (UNLIKELY(shift != home_shift) && + home != BottomNBits(h->hashed_key[1], home_shift)) { + // This chain is in a Grow operation and we've landed on an entry + // that belongs to the wrong destination chain. We can keep going, but + // there's a chance we'll need to backtrack back *before* this entry, + // if the Grow finishes before this Lookup. We cannot save this entry + // for backtracking because it might soon or already be on the wrong + // chain. + // NOTE: if we simply backtrack rather than continuing, we would + // be in a wait loop (not allowed in Lookup!) until the other thread + // finishes its Grow. + Unref(*h); + } else { + // Correct home location, so we are on the right chain. + // With new usable read ref, can release old one (if applicable). + if (read_ref_on_chain) { + // Pretend we never took the reference. + Unref(*read_ref_on_chain); + } + // And keep the new one. + read_ref_on_chain = h; + } + } else { + if (full_match_or_unknown) { + // Must have been an "under construction" entry. Can safely skip it, + // but there's a chance we'll have to backtrack later + } else { + // Home mismatch! Revert back to a safe point (if we have it) + h = read_ref_on_chain; + // Didn't make progress & retry + } + } + } +} + +void AutoHyperClockTable::Remove(HandleImpl* h) { + assert((h->meta.Load() >> ClockHandle::kStateShift) == + ClockHandle::kStateConstruction); + + const HandleImpl& c_h = *h; + PurgeImpl(&c_h.hashed_key); +} + +bool AutoHyperClockTable::TryEraseHandle(HandleImpl* h, bool holding_ref, + bool mark_invisible) { + uint64_t meta; + if (mark_invisible) { + // Set invisible + meta = h->meta.FetchAnd( + ~(uint64_t{ClockHandle::kStateVisibleBit} << ClockHandle::kStateShift)); + // To local variable also + meta &= + ~(uint64_t{ClockHandle::kStateVisibleBit} << ClockHandle::kStateShift); + } else { + meta = h->meta.Load(); + } + + // Take ownership if no other refs + do { + if (GetRefcount(meta) != uint64_t{holding_ref}) { + // Not last ref at some point in time during this call + return false; + } + if ((meta & (uint64_t{ClockHandle::kStateShareableBit} + << ClockHandle::kStateShift)) == 0) { + // Someone else took ownership + return false; + } + // Note that if !holding_ref, there's a small chance that we release, + // another thread replaces this entry with another, reaches zero refs, and + // then we end up erasing that other entry. That's an acceptable risk / + // imprecision. + } while (!h->meta.CasWeak(meta, uint64_t{ClockHandle::kStateConstruction} + << ClockHandle::kStateShift)); + // Took ownership + // TODO? Delay freeing? + h->FreeData(allocator_); + size_t total_charge = h->total_charge; + if (UNLIKELY(h->IsStandalone())) { + // Delete detached handle + delete h; + standalone_usage_.FetchSubRelaxed(total_charge); + } else { + Remove(h); + MarkEmpty(*h); + occupancy_.FetchSub(1U); + } + usage_.FetchSubRelaxed(total_charge); + assert(usage_.LoadRelaxed() < SIZE_MAX / 2); + return true; +} + +bool AutoHyperClockTable::Release(HandleImpl* h, bool useful, + bool erase_if_last_ref) { + // In contrast with LRUCache's Release, this function won't delete the handle + // when the cache is above capacity and the reference is the last one. Space + // is only freed up by Evict/PurgeImpl (called by Insert when space + // is needed) and Erase. We do this to avoid an extra atomic read of the + // variable usage_. + + uint64_t old_meta; + if (useful) { + // Increment release counter to indicate was used + old_meta = h->meta.FetchAdd(ClockHandle::kReleaseIncrement); + // Correct for possible (but rare) overflow + CorrectNearOverflow(old_meta, h->meta); + } else { + // Decrement acquire counter to pretend it never happened + old_meta = h->meta.FetchSub(ClockHandle::kAcquireIncrement); + } + + assert((old_meta >> ClockHandle::kStateShift) & + ClockHandle::kStateShareableBit); + // No underflow + assert(((old_meta >> ClockHandle::kAcquireCounterShift) & + ClockHandle::kCounterMask) != + ((old_meta >> ClockHandle::kReleaseCounterShift) & + ClockHandle::kCounterMask)); + + if ((erase_if_last_ref || UNLIKELY(old_meta >> ClockHandle::kStateShift == + ClockHandle::kStateInvisible))) { + // FIXME: There's a chance here that another thread could replace this + // entry and we end up erasing the wrong one. + return TryEraseHandle(h, /*holding_ref=*/false, /*mark_invisible=*/false); + } else { + return false; + } +} + +#ifndef NDEBUG +void AutoHyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) { + if (n > 0) { + // Do n-1 simple releases first + TEST_ReleaseNMinus1(h, n); + + // Then the last release might be more involved + Release(h, /*useful*/ true, /*erase_if_last_ref*/ false); + } +} +#endif + +void AutoHyperClockTable::Erase(const UniqueId64x2& hashed_key) { + // Don't need to be efficient. + // Might be one match masking another, so loop. + while (HandleImpl* h = Lookup(hashed_key)) { + bool gone = + TryEraseHandle(h, /*holding_ref=*/true, /*mark_invisible=*/true); + if (!gone) { + // Only marked invisible, which is ok. + // Pretend we never took the reference from Lookup. + Unref(*h); + } + } +} + +void AutoHyperClockTable::EraseUnRefEntries() { + size_t usable_size = GetTableSize(); + for (size_t i = 0; i < usable_size; i++) { + HandleImpl& h = array_[i]; + + uint64_t old_meta = h.meta.LoadRelaxed(); + if (old_meta & (uint64_t{ClockHandle::kStateShareableBit} + << ClockHandle::kStateShift) && + GetRefcount(old_meta) == 0 && + h.meta.CasStrong(old_meta, uint64_t{ClockHandle::kStateConstruction} + << ClockHandle::kStateShift)) { + // Took ownership + h.FreeData(allocator_); + usage_.FetchSubRelaxed(h.total_charge); + // NOTE: could be more efficient with a dedicated variant of + // PurgeImpl, but this is not a common operation + Remove(&h); + MarkEmpty(h); + occupancy_.FetchSub(1U); + } + } +} + +void AutoHyperClockTable::Evict(size_t requested_charge, InsertState& state, + EvictionData* data, + uint32_t eviction_effort_cap) { + // precondition + assert(requested_charge > 0); + + // We need the clock pointer to seemlessly "wrap around" at the end of the + // table, and to be reasonably stable under Grow operations. This is + // challenging when the linear hashing progressively opens additional + // most-significant-hash-bits in determining home locations. + + // TODO: make a tuning parameter? + // Up to 2x this number of homes will be evicted per step. In very rare + // cases, possibly more, as homes of an out-of-date generation will be + // resolved to multiple in a newer generation. + constexpr size_t step_size = 4; + + // A clock_pointer_mask_ field separate from length_info_ enables us to use + // the same mask (way of dividing up the space among evicting threads) for + // iterating over the whole structure before considering changing the mask + // at the beginning of each pass. This ensures we do not have a large portion + // of the space that receives redundant or missed clock updates. However, + // with two variables, for each update to clock_pointer_mask (< 64 ever in + // the life of the cache), there will be a brief period where concurrent + // eviction threads could use the old mask value, possibly causing redundant + // or missed clock updates for a *small* portion of the table. + size_t clock_pointer_mask = clock_pointer_mask_.LoadRelaxed(); + + uint64_t max_clock_pointer = 0; // unset + + // TODO: consider updating during a long eviction + size_t used_length = LengthInfoToUsedLength(state.saved_length_info); + + autovector to_finish_eviction; + + // Loop until enough freed, or limit reached (see bottom of loop) + for (;;) { + // First (concurrent) increment clock pointer + uint64_t old_clock_pointer = clock_pointer_.FetchAddRelaxed(step_size); + + if (UNLIKELY((old_clock_pointer & clock_pointer_mask) == 0)) { + // Back at the beginning. See if clock_pointer_mask should be updated. + uint64_t mask = BottomNBits( + UINT64_MAX, LengthInfoToMinShift(state.saved_length_info)); + if (clock_pointer_mask != mask) { + clock_pointer_mask = static_cast(mask); + clock_pointer_mask_.StoreRelaxed(clock_pointer_mask); + } + } + + size_t major_step = clock_pointer_mask + 1; + assert((major_step & clock_pointer_mask) == 0); + + for (size_t base_home = old_clock_pointer & clock_pointer_mask; + base_home < used_length; base_home += major_step) { + for (size_t i = 0; i < step_size; i++) { + size_t home = base_home + i; + if (home >= used_length) { + break; + } + PurgeImpl(&to_finish_eviction, home, data); + } + } + + for (HandleImpl* h : to_finish_eviction) { + TrackAndReleaseEvictedEntry(h); + // NOTE: setting likely_empty_slot here can cause us to reduce the + // portion of "at home" entries, probably because an evicted entry + // is more likely to come back than a random new entry and would be + // unable to go into its home slot. + } + to_finish_eviction.clear(); + + // Loop exit conditions + if (data->freed_charge >= requested_charge) { + return; + } + + if (max_clock_pointer == 0) { + // Cap the eviction effort at this thread (along with those operating in + // parallel) circling through the whole structure kMaxCountdown times. + // In other words, this eviction run must find something/anything that is + // unreferenced at start of and during the eviction run that isn't + // reclaimed by a concurrent eviction run. + // TODO: Does HyperClockCache need kMaxCountdown + 1? + max_clock_pointer = + old_clock_pointer + + (uint64_t{ClockHandle::kMaxCountdown + 1} * major_step); + } + + if (old_clock_pointer + step_size >= max_clock_pointer) { + return; + } + + if (IsEvictionEffortExceeded(*data, eviction_effort_cap)) { + eviction_effort_exceeded_count_.FetchAddRelaxed(1); + return; + } + } +} + +size_t AutoHyperClockTable::CalcMaxUsableLength( + size_t capacity, size_t min_avg_value_size, + CacheMetadataChargePolicy metadata_charge_policy) { + double min_avg_slot_charge = min_avg_value_size * kMaxLoadFactor; + if (metadata_charge_policy == kFullChargeCacheMetadata) { + min_avg_slot_charge += sizeof(HandleImpl); + } + assert(min_avg_slot_charge > 0.0); + size_t num_slots = + static_cast(capacity / min_avg_slot_charge + 0.999999); + + const size_t slots_per_page = port::kPageSize / sizeof(HandleImpl); + + // Round up to page size + return ((num_slots + slots_per_page - 1) / slots_per_page) * slots_per_page; +} + +namespace { +bool IsHeadNonempty(const AutoHyperClockTable::HandleImpl& h) { + return !AutoHyperClockTable::HandleImpl::IsEnd( + h.head_next_with_shift.LoadRelaxed()); +} +bool IsEntryAtHome(const AutoHyperClockTable::HandleImpl& h, int shift, + size_t home) { + if (MatchAndRef(nullptr, h, shift, home)) { + Unref(h); + return true; + } else { + return false; + } +} +} // namespace + +void AutoHyperClockCache::ReportProblems( + const std::shared_ptr& info_log) const { + BaseHyperClockCache::ReportProblems(info_log); + + if (info_log->GetInfoLogLevel() <= InfoLogLevel::DEBUG_LEVEL) { + LoadVarianceStats head_stats; + size_t entry_at_home_count = 0; + uint64_t yield_count = 0; + this->ForEachShard([&](const Shard* shard) { + size_t count = shard->GetTableAddressCount(); + uint64_t length_info = UsedLengthToLengthInfo(count); + for (size_t i = 0; i < count; ++i) { + const auto& h = *shard->GetTable().HandlePtr(i); + head_stats.Add(IsHeadNonempty(h)); + int shift; + size_t home; + GetHomeIndexAndShift(length_info, i, &home, &shift); + assert(home == i); + entry_at_home_count += IsEntryAtHome(h, shift, home); + } + yield_count += shard->GetTable().GetYieldCount(); + }); + ROCKS_LOG_AT_LEVEL(info_log, InfoLogLevel::DEBUG_LEVEL, + "Head occupancy stats: %s", head_stats.Report().c_str()); + ROCKS_LOG_AT_LEVEL(info_log, InfoLogLevel::DEBUG_LEVEL, + "Entries at home count: %zu", entry_at_home_count); + ROCKS_LOG_AT_LEVEL(info_log, InfoLogLevel::DEBUG_LEVEL, + "Yield count: %" PRIu64, yield_count); + } +} + +} // namespace clock_cache + +// DEPRECATED (see public API) +std::shared_ptr NewClockCache( + size_t capacity, int num_shard_bits, bool strict_capacity_limit, + CacheMetadataChargePolicy metadata_charge_policy) { + return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit, + /* high_pri_pool_ratio */ 0.5, nullptr, + kDefaultToAdaptiveMutex, metadata_charge_policy, + /* low_pri_pool_ratio */ 0.0); +} + +std::shared_ptr HyperClockCacheOptions::MakeSharedCache() const { + // For sanitized options + HyperClockCacheOptions opts = *this; + if (opts.num_shard_bits >= 20) { + return nullptr; // The cache cannot be sharded into too many fine pieces. + } + if (opts.num_shard_bits < 0) { + // Use larger shard size to reduce risk of large entries clustering + // or skewing individual shards. + constexpr size_t min_shard_size = 32U * 1024U * 1024U; + opts.num_shard_bits = + GetDefaultCacheShardBits(opts.capacity, min_shard_size); + } + std::shared_ptr cache; + if (opts.estimated_entry_charge == 0) { + cache = std::make_shared(opts); + } else { + cache = std::make_shared(opts); } - std::shared_ptr cache = - std::make_shared(opts); if (opts.secondary_cache) { cache = std::make_shared(cache, opts.secondary_cache); diff --git a/cache/clock_cache.h b/cache/clock_cache.h index fee2eb11a..54d656021 100644 --- a/cache/clock_cache.h +++ b/cache/clock_cache.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -20,16 +21,20 @@ #include "cache/sharded_cache.h" #include "port/lang.h" #include "port/malloc.h" +#include "port/mmap.h" #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/secondary_cache.h" +#include "util/atomic.h" #include "util/autovector.h" +#include "util/math.h" namespace ROCKSDB_NAMESPACE { namespace clock_cache { // Forward declaration of friend class. +template class ClockCacheTest; // HyperClockCache is an alternative to LRUCache specifically tailored for @@ -37,24 +42,31 @@ class ClockCacheTest; // // Benefits // -------- -// * Fully lock free (no waits or spins) for efficiency under high concurrency +// * Lock/wait free (no waits or spins) for efficiency under high concurrency +// * Fixed version (estimated_entry_charge > 0) is fully lock/wait free +// * Automatic version (estimated_entry_charge = 0) has rare waits among +// certain insertion or erase operations that involve the same very small +// set of entries. // * Optimized for hot path reads. For concurrency control, most Lookup() and // essentially all Release() are a single atomic add operation. -// * Eviction on insertion is fully parallel and lock-free. +// * Eviction on insertion is fully parallel. // * Uses a generalized + aging variant of CLOCK eviction that might outperform // LRU in some cases. (For background, see // https://en.wikipedia.org/wiki/Page_replacement_algorithm) // // Costs // ----- -// * Hash table is not resizable (for lock-free efficiency) so capacity is not -// dynamically changeable. Rely on an estimated average value (block) size for +// * FixedHyperClockCache (estimated_entry_charge > 0) - Hash table is not +// resizable (for lock-free efficiency) so capacity is not dynamically +// changeable. Rely on an estimated average value (block) size for // space+time efficiency. (See estimated_entry_charge option details.) +// EXPERIMENTAL - This limitation is fixed in AutoHyperClockCache, activated +// with estimated_entry_charge == 0. // * Insert usually does not (but might) overwrite a previous entry associated -// with a cache key. This is OK for RocksDB uses of Cache. +// with a cache key. This is OK for RocksDB uses of Cache, though it does mess +// up our REDUNDANT block cache insertion statistics. // * Only supports keys of exactly 16 bytes, which is what RocksDB uses for -// block cache (not row cache or table cache). -// * SecondaryCache is not supported. +// block cache (but not row cache or table cache). // * Cache priorities are less aggressively enforced. Unlike LRUCache, enough // transient LOW or BOTTOM priority items can evict HIGH priority entries that // are not referenced recently (or often) enough. @@ -137,7 +149,8 @@ class ClockCacheTest; // * Empty - slot is not in use and unowned. All other metadata and data is // in an undefined state. // * Construction - slot is exclusively owned by one thread, the thread -// successfully entering this state, for populating or freeing data. +// successfully entering this state, for populating or freeing data +// (de-construction, same state marker). // * Shareable (group) - slot holds an entry with counted references for // pinning and reading, including // * Visible - slot holds an entry that can be returned by Lookup @@ -185,15 +198,19 @@ class ClockCacheTest; // know from our "redundant" stats that overwrites are very rare for the block // cache, so we should not spend much to make them effective. // -// So instead we Insert as soon as we find an empty slot in the probing -// sequence without seeing an existing (visible) entry for the same key. This -// way we only insert if we can improve the probing performance, and we don't -// need to probe beyond our insert position, assuming we are willing to let -// the previous entry for the same key die of old age (eventual eviction from -// not being used). We can reach a similar state with concurrent insertions, -// where one will pass over the other while it is "under construction." -// This temporary duplication is acceptable for RocksDB block cache because -// we know redundant insertion is rare. +// FixedHyperClockCache: Instead we Insert as soon as we find an empty slot in +// the probing sequence without seeing an existing (visible) entry for the same +// key. This way we only insert if we can improve the probing performance, and +// we don't need to probe beyond our insert position, assuming we are willing +// to let the previous entry for the same key die of old age (eventual eviction +// from not being used). We can reach a similar state with concurrent +// insertions, where one will pass over the other while it is "under +// construction." This temporary duplication is acceptable for RocksDB block +// cache because we know redundant insertion is rare. +// AutoHyperClockCache: Similar, except we only notice and return an existing +// match if it is found in the search for a suitable empty slot (starting with +// the same slot as the head pointer), not by following the existing chain of +// entries. Insertions are always made to the head of the chain. // // Another problem to solve is what to return to the caller when we find an // existing entry whose probing position we cannot improve on, or when the @@ -281,29 +298,6 @@ class ClockCacheTest; // ----------------------------------------------------------------------- // -// The load factor p is a real number in (0, 1) such that at all -// times at most a fraction p of all slots, without counting tombstones, -// are occupied by elements. This means that the probability that a random -// probe hits an occupied slot is at most p, and thus at most 1/p probes -// are required on average. For example, p = 70% implies that between 1 and 2 -// probes are needed on average (bear in mind that this reasoning doesn't -// consider the effects of clustering over time, which should be negligible -// with double hashing). -// Because the size of the hash table is always rounded up to the next -// power of 2, p is really an upper bound on the actual load factor---the -// actual load factor is anywhere between p/2 and p. This is a bit wasteful, -// but bear in mind that slots only hold metadata, not actual values. -// Since space cost is dominated by the values (the LSM blocks), -// overprovisioning the table with metadata only increases the total cache space -// usage by a tiny fraction. -constexpr double kLoadFactor = 0.7; - -// The user can exceed kLoadFactor if the sizes of the inserted values don't -// match estimated_value_size, or in some rare cases with -// strict_capacity_limit == false. To avoid degenerate performance, we set a -// strict upper bound on the load factor. -constexpr double kStrictLoadFactor = 0.84; - struct ClockHandleBasicData { Cache::ObjectPtr value = nullptr; const Cache::CacheItemHelper* helper = nullptr; @@ -326,7 +320,7 @@ struct ClockHandle : public ClockHandleBasicData { // state of the handle. The meta word looks like this: // low bits high bits // ----------------------------------------------------------------------- - // | acquire counter | release counter | state marker | + // | acquire counter | release counter | hit bit | state marker | // ----------------------------------------------------------------------- // For reading or updating counters in meta word. @@ -340,8 +334,12 @@ struct ClockHandle : public ClockHandleBasicData { static constexpr uint64_t kReleaseIncrement = uint64_t{1} << kReleaseCounterShift; + // For setting the hit bit + static constexpr uint8_t kHitBitShift = 2U * kCounterNumBits; + static constexpr uint64_t kHitBitMask = uint64_t{1} << kHitBitShift; + // For reading or updating the state marker in meta word - static constexpr uint8_t kStateShift = 2U * kCounterNumBits; + static constexpr uint8_t kStateShift = kHitBitShift + 1; // Bits contribution to state marker. // Occupied means any state other than empty @@ -372,22 +370,19 @@ struct ClockHandle : public ClockHandleBasicData { // TODO: make these coundown values tuning parameters for eviction? // See above. Mutable for read reference counting. - mutable std::atomic meta{}; - - // Whether this is a "deteched" handle that is independently allocated - // with `new` (so must be deleted with `delete`). - // TODO: ideally this would be packed into some other data field, such - // as upper bits of total_charge, but that incurs a measurable performance - // regression. - bool standalone = false; - - inline bool IsStandalone() const { return standalone; } - - inline void SetStandalone() { standalone = true; } + mutable AcqRelAtomic meta{}; }; // struct ClockHandle class BaseClockTable { public: + struct BaseOpts { + explicit BaseOpts(int _eviction_effort_cap) + : eviction_effort_cap(_eviction_effort_cap) {} + explicit BaseOpts(const HyperClockCacheOptions& opts) + : BaseOpts(opts.eviction_effort_cap) {} + int eviction_effort_cap; + }; + BaseClockTable(CacheMetadataChargePolicy metadata_charge_policy, MemoryAllocator* allocator, const Cache::EvictionCallback* eviction_callback, @@ -400,28 +395,38 @@ class BaseClockTable { template typename Table::HandleImpl* CreateStandalone(ClockHandleBasicData& proto, size_t capacity, - bool strict_capacity_limit, + uint32_t eec_and_scl, bool allow_uncharged); template Status Insert(const ClockHandleBasicData& proto, typename Table::HandleImpl** handle, Cache::Priority priority, - size_t capacity, bool strict_capacity_limit); + size_t capacity, uint32_t eec_and_scl); void Ref(ClockHandle& handle); - size_t GetOccupancy() const { - return occupancy_.load(std::memory_order_relaxed); - } + size_t GetOccupancy() const { return occupancy_.LoadRelaxed(); } - size_t GetUsage() const { return usage_.load(std::memory_order_relaxed); } + size_t GetUsage() const { return usage_.LoadRelaxed(); } - size_t GetStandaloneUsage() const { - return standalone_usage_.load(std::memory_order_relaxed); - } + size_t GetStandaloneUsage() const { return standalone_usage_.LoadRelaxed(); } uint32_t GetHashSeed() const { return hash_seed_; } + uint64_t GetYieldCount() const { return yield_count_.LoadRelaxed(); } + + uint64_t GetEvictionEffortExceededCount() const { + return eviction_effort_exceeded_count_.LoadRelaxed(); + } + + struct EvictionData { + size_t freed_charge = 0; + size_t freed_count = 0; + size_t seen_pinned_count = 0; + }; + + void TrackAndReleaseEvictedEntry(ClockHandle* h); + #ifndef NDEBUG // Acquire N references void TEST_RefN(ClockHandle& handle, size_t n); @@ -445,6 +450,7 @@ class BaseClockTable { template Status ChargeUsageMaybeEvictStrict(size_t total_charge, size_t capacity, bool need_evict_for_occupancy, + uint32_t eviction_effort_cap, typename Table::InsertState& state); // Helper for updating `usage_` for new entry with given `total_charge` @@ -458,6 +464,7 @@ class BaseClockTable { template bool ChargeUsageMaybeEvictNonStrict(size_t total_charge, size_t capacity, bool need_evict_for_occupancy, + uint32_t eviction_effort_cap, typename Table::InsertState& state); protected: // data @@ -466,17 +473,29 @@ class BaseClockTable { // operations in ClockCacheShard. // Clock algorithm sweep pointer. - std::atomic clock_pointer_{}; + // (Relaxed: only needs to be consistent with itself.) + RelaxedAtomic clock_pointer_{}; + + // Counter for number of times we yield to wait on another thread. + // It is normal for this to occur rarely in normal operation. + // (Relaxed: a simple stat counter.) + RelaxedAtomic yield_count_{}; + + // Counter for number of times eviction effort cap is exceeded. + // It is normal for this to occur rarely in normal operation. + // (Relaxed: a simple stat counter.) + RelaxedAtomic eviction_effort_exceeded_count_{}; + // TODO: is this separation needed if we don't do background evictions? ALIGN_AS(CACHE_LINE_SIZE) // Number of elements in the table. - std::atomic occupancy_{}; + AcqRelAtomic occupancy_{}; // Memory usage by entries tracked by the cache (including standalone) - std::atomic usage_{}; + AcqRelAtomic usage_{}; // Part of usage by standalone entries (not in table) - std::atomic standalone_usage_{}; + AcqRelAtomic standalone_usage_{}; ALIGN_AS(CACHE_LINE_SIZE) const CacheMetadataChargePolicy metadata_charge_policy_; @@ -491,27 +510,53 @@ class BaseClockTable { const uint32_t& hash_seed_; }; -class HyperClockTable : public BaseClockTable { +// Hash table for cache entries with size determined at creation time. +// Uses open addressing and double hashing. Since entries cannot be moved, +// the "displacements" count ensures probing sequences find entries even when +// entries earlier in the probing sequence have been removed. +class FixedHyperClockTable : public BaseClockTable { public: // Target size to be exactly a common cache line size (see static_assert in // clock_cache.cc) struct ALIGN_AS(64U) HandleImpl : public ClockHandle { // The number of elements that hash to this slot or a lower one, but wind // up in this slot or a higher one. - std::atomic displacements{}; - + // (Relaxed: within a Cache op, does not need consistency with entries + // inserted/removed during that op. For example, a Lookup() that + // happens-after an Insert() will see an appropriate displacements value + // for the entry to be in a published state.) + RelaxedAtomic displacements{}; + + // Whether this is a "deteched" handle that is independently allocated + // with `new` (so must be deleted with `delete`). + // TODO: ideally this would be packed into some other data field, such + // as upper bits of total_charge, but that incurs a measurable performance + // regression. + bool standalone = false; + + inline bool IsStandalone() const { return standalone; } + + inline void SetStandalone() { standalone = true; } }; // struct HandleImpl - struct Opts { + struct Opts : public BaseOpts { + explicit Opts(size_t _estimated_value_size, int _eviction_effort_cap) + : BaseOpts(_eviction_effort_cap), + estimated_value_size(_estimated_value_size) {} + explicit Opts(const HyperClockCacheOptions& opts) + : BaseOpts(opts.eviction_effort_cap) { + assert(opts.estimated_entry_charge > 0); + estimated_value_size = opts.estimated_entry_charge; + } size_t estimated_value_size; }; - HyperClockTable(size_t capacity, bool strict_capacity_limit, - CacheMetadataChargePolicy metadata_charge_policy, - MemoryAllocator* allocator, - const Cache::EvictionCallback* eviction_callback, - const uint32_t* hash_seed, const Opts& opts); - ~HyperClockTable(); + FixedHyperClockTable(size_t capacity, + CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback, + const uint32_t* hash_seed, const Opts& opts); + ~FixedHyperClockTable(); // For BaseClockTable::Insert struct InsertState {}; @@ -528,8 +573,8 @@ class HyperClockTable : public BaseClockTable { // Runs the clock eviction algorithm trying to reclaim at least // requested_charge. Returns how much is evicted, which could be less // if it appears impossible to evict the requested amount without blocking. - void Evict(size_t requested_charge, size_t* freed_charge, size_t* freed_count, - InsertState& state); + void Evict(size_t requested_charge, InsertState& state, EvictionData* data, + uint32_t eviction_effort_cap); HandleImpl* Lookup(const UniqueId64x2& hashed_key); @@ -546,7 +591,7 @@ class HyperClockTable : public BaseClockTable { const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; } #ifndef NDEBUG - size_t& TEST_MutableOccupancyLimit() const { + size_t& TEST_MutableOccupancyLimit() { return const_cast(occupancy_limit_); } @@ -554,10 +599,33 @@ class HyperClockTable : public BaseClockTable { void TEST_ReleaseN(HandleImpl* handle, size_t n); #endif + // The load factor p is a real number in (0, 1) such that at all + // times at most a fraction p of all slots, without counting tombstones, + // are occupied by elements. This means that the probability that a random + // probe hits an occupied slot is at most p, and thus at most 1/p probes + // are required on average. For example, p = 70% implies that between 1 and 2 + // probes are needed on average (bear in mind that this reasoning doesn't + // consider the effects of clustering over time, which should be negligible + // with double hashing). + // Because the size of the hash table is always rounded up to the next + // power of 2, p is really an upper bound on the actual load factor---the + // actual load factor is anywhere between p/2 and p. This is a bit wasteful, + // but bear in mind that slots only hold metadata, not actual values. + // Since space cost is dominated by the values (the LSM blocks), + // overprovisioning the table with metadata only increases the total cache + // space usage by a tiny fraction. + static constexpr double kLoadFactor = 0.7; + + // The user can exceed kLoadFactor if the sizes of the inserted values don't + // match estimated_value_size, or in some rare cases with + // strict_capacity_limit == false. To avoid degenerate performance, we set a + // strict upper bound on the load factor. + static constexpr double kStrictLoadFactor = 0.84; + private: // functions // Returns x mod 2^{length_bits_}. inline size_t ModTableSize(uint64_t x) { - return static_cast(x) & length_bits_mask_; + return BitwiseAnd(x, length_bits_mask_); } // Returns the first slot in the probe sequence with a handle e such that @@ -570,8 +638,9 @@ class HyperClockTable : public BaseClockTable { // slot probed. This function uses templates instead of std::function to // minimize the risk of heap-allocated closures being created. template - inline HandleImpl* FindSlot(const UniqueId64x2& hashed_key, MatchFn match_fn, - AbortFn abort_fn, UpdateFn update_fn); + inline HandleImpl* FindSlot(const UniqueId64x2& hashed_key, + const MatchFn& match_fn, const AbortFn& abort_fn, + const UpdateFn& update_fn); // Re-decrement all displacements in probe path starting from beginning // until (not including) the given handle @@ -604,12 +673,329 @@ class HyperClockTable : public BaseClockTable { // Array of slots comprising the hash table. const std::unique_ptr array_; -}; // class HyperClockTable +}; // class FixedHyperClockTable + +// Hash table for cache entries that resizes automatically based on occupancy. +// However, it depends on a contiguous memory region to grow into +// incrementally, using linear hashing, so uses an anonymous mmap so that +// only the used portion of the memory region is mapped to physical memory +// (part of RSS). +// +// This table implementation uses the same "low-level protocol" for managing +// the contens of an entry slot as FixedHyperClockTable does, captured in the +// ClockHandle struct. The provides most of the essential data safety, but +// AutoHyperClockTable is another "high-level protocol" for organizing entries +// into a hash table, with automatic resizing. +// +// This implementation is not fully wait-free but we can call it "essentially +// wait-free," and here's why. First, like FixedHyperClockCache, there is no +// locking nor other forms of waiting at the cache or shard level. Also like +// FixedHCC there is essentially an entry-level read-write lock implemented +// with atomics, but our relaxed atomicity/consistency guarantees (e.g. +// duplicate inserts are possible) mean we do not need to wait for entry +// locking. Lookups, non-erasing Releases, and non-evicting non-growing Inserts +// are all fully wait-free. Of course, these waits are not dependent on any +// external factors such as I/O. +// +// For operations that remove entries from a chain or grow the table by +// splitting a chain, there is a chain-level locking mechanism that we call a +// "rewrite" lock, and the only waits are for these locks. On average, each +// chain lock is relevant to < 2 entries each. (The average would be less than +// one entry each, but we do not lock when there's no entry to remove or +// migrate.) And a given thread can only hold two such chain locks at a time, +// more typically just one. So in that sense alone, the waiting that does exist +// is very localized. +// +// If we look closer at the operations utilizing that locking mechanism, we +// can see why it's "essentially wait-free." +// * Grow operations to increase the size of the table: each operation splits +// an existing chain into two, and chains for splitting are chosen in table +// order. Grow operations are fully parallel except for the chain locking, but +// for one Grow operation to wait on another, it has to be feeding into the +// other, which means the table has doubled in size already from other Grow +// operations without the original one finishing. So Grow operations are very +// low latency (unlike LRUCache doubling the table size in one operation) and +// very parallelizeable. (We use some tricks to break up dependencies in +// updating metadata on the usable size of the table.) And obviously Grow +// operations are very rare after the initial population of the table. +// * Evict operations (part of many Inserts): clock updates and evictions +// sweep through the structure in table order, so like Grow operations, +// parallel Evict can only wait on each other if an Evict has lingered (slept) +// long enough that the clock pointer has wrapped around the entire structure. +// * Random erasures (Erase, Release with erase_if_last_ref, etc.): these +// operations are rare and not really considered performance critical. +// Currently they're mostly used for removing placeholder cache entries, e.g. +// for memory tracking, though that could use standalone entries instead to +// avoid potential contention in table operations. It's possible that future +// enhancements could pro-actively remove cache entries from obsolete files, +// but that's not yet implemented. +class AutoHyperClockTable : public BaseClockTable { + public: + // Target size to be exactly a common cache line size (see static_assert in + // clock_cache.cc) + struct ALIGN_AS(64U) HandleImpl : public ClockHandle { + // To orgainize AutoHyperClockTable entries into a hash table while + // allowing the table size to grow without existing entries being moved, + // a version of chaining is used. Rather than being heap allocated (and + // incurring overheads to ensure memory safety) entries must go into + // Handles ("slots") in the pre-allocated array. To improve CPU cache + // locality, the chain head pointers are interleved with the entries; + // specifically, a Handle contains + // * A head pointer for a chain of entries with this "home" location. + // * A ClockHandle, for an entry that may or may not be in the chain + // starting from that head (but for performance ideally is on that + // chain). + // * A next pointer for the continuation of the chain containing this + // entry. + // + // The pointers are not raw pointers, but are indices into the array, + // and are decorated in two ways to help detect and recover from + // relevant concurrent modifications during Lookup, so that Lookup is + // fully wait-free: + // * Each "with_shift" pointer contains a shift count that indicates + // how many hash bits were used in chosing the home address for the + // chain--specifically the next entry in the chain. + // * The end of a chain is given a special "end" marker and refers back + // to the head of the chain. + // + // Why do we need shift on each pointer? To make Lookup wait-free, we need + // to be able to query a chain without missing anything, and preferably + // avoid synchronously double-checking the length_info. Without the shifts, + // there is a risk that we start down a chain and while paused on an entry + // that goes to a new home, we then follow the rest of the + // partially-migrated chain to see the shared ending with the old home, but + // for a time were following the chain for the new home, missing some + // entries for the old home. + // + // Why do we need the end of the chain to loop back? If Lookup pauses + // at an "under construction" entry, and sees that "next" is null after + // waking up, we need something to tell whether the "under construction" + // entry was freed and reused for another chain. Otherwise, we could + // miss entries still on the original chain due in the presence of a + // concurrent modification. Until an entry is fully erased from a chain, + // it is normal to see "under construction" entries on the chain, and it + // is not safe to read their hashed key without either a read reference + // on the entry or a rewrite lock on the chain. + + // Marker in a "with_shift" head pointer for some thread owning writes + // to the chain structure (except for inserts), but only if not an + // "end" pointer. Also called the "rewrite lock." + static constexpr uint64_t kHeadLocked = uint64_t{1} << 7; + + // Marker in a "with_shift" pointer for the end of a chain. Must also + // point back to the head of the chain (with end marker removed). + // Also includes the "locked" bit so that attempting to lock an empty + // chain has no effect (not needed, as the lock is only needed for + // removals). + static constexpr uint64_t kNextEndFlags = (uint64_t{1} << 6) | kHeadLocked; + + static inline bool IsEnd(uint64_t next_with_shift) { + // Assuming certain values never used, suffices to check this one bit + constexpr auto kCheckBit = kNextEndFlags ^ kHeadLocked; + return next_with_shift & kCheckBit; + } + + // Bottom bits to right shift away to get an array index from a + // "with_shift" pointer. + static constexpr int kNextShift = 8; + + // A bit mask for the "shift" associated with each "with_shift" pointer. + // Always bottommost bits. + static constexpr int kShiftMask = 63; + + // A marker for head_next_with_shift that indicates this HandleImpl is + // heap allocated (standalone) rather than in the table. + static constexpr uint64_t kStandaloneMarker = UINT64_MAX; + + // A marker for head_next_with_shift indicating the head is not yet part + // of the usable table, or for chain_next_with_shift indicating that the + // entry is not present or is not yet part of a chain (must not be + // "shareable" state). + static constexpr uint64_t kUnusedMarker = 0; + + // See above. The head pointer is logically independent of the rest of + // the entry, including the chain next pointer. + AcqRelAtomic head_next_with_shift{kUnusedMarker}; + AcqRelAtomic chain_next_with_shift{kUnusedMarker}; + + // For supporting CreateStandalone and some fallback cases. + inline bool IsStandalone() const { + return head_next_with_shift.Load() == kStandaloneMarker; + } + + inline void SetStandalone() { + head_next_with_shift.Store(kStandaloneMarker); + } + }; // struct HandleImpl + + struct Opts : public BaseOpts { + explicit Opts(size_t _min_avg_value_size, int _eviction_effort_cap) + : BaseOpts(_eviction_effort_cap), + min_avg_value_size(_min_avg_value_size) {} + + explicit Opts(const HyperClockCacheOptions& opts) + : BaseOpts(opts.eviction_effort_cap) { + assert(opts.estimated_entry_charge == 0); + min_avg_value_size = opts.min_avg_entry_charge; + } + size_t min_avg_value_size; + }; + + AutoHyperClockTable(size_t capacity, + CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback, + const uint32_t* hash_seed, const Opts& opts); + ~AutoHyperClockTable(); + + // For BaseClockTable::Insert + struct InsertState { + uint64_t saved_length_info = 0; + size_t likely_empty_slot = 0; + }; + + void StartInsert(InsertState& state); + + // Does initial check for whether there's hash table room for another + // inserted entry, possibly growing if needed. Returns true iff (after + // the call) there is room for the proposed number of entries. + bool GrowIfNeeded(size_t new_occupancy, InsertState& state); + + HandleImpl* DoInsert(const ClockHandleBasicData& proto, + uint64_t initial_countdown, bool take_ref, + InsertState& state); + + // Runs the clock eviction algorithm trying to reclaim at least + // requested_charge. Returns how much is evicted, which could be less + // if it appears impossible to evict the requested amount without blocking. + void Evict(size_t requested_charge, InsertState& state, EvictionData* data, + uint32_t eviction_effort_cap); + + HandleImpl* Lookup(const UniqueId64x2& hashed_key); + + bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref); + + void Erase(const UniqueId64x2& hashed_key); + + void EraseUnRefEntries(); + + size_t GetTableSize() const; + + size_t GetOccupancyLimit() const; + + const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; } + +#ifndef NDEBUG + size_t& TEST_MutableOccupancyLimit() { + return *reinterpret_cast(&occupancy_limit_); + } + + // Release N references + void TEST_ReleaseN(HandleImpl* handle, size_t n); +#endif + + // Maximum ratio of number of occupied slots to number of usable slots. The + // actual load factor should float pretty close to this number, which should + // be a nice space/time trade-off, though large swings in WriteBufferManager + // memory could lead to low (but very much safe) load factors (only after + // seeing high load factors). Linear hashing along with (modified) linear + // probing to find an available slot increases potential risks of high + // load factors, so are disallowed. + static constexpr double kMaxLoadFactor = 0.60; + + private: // functions + // Returns true iff increased usable length. Due to load factor + // considerations, GrowIfNeeded might call this more than once to make room + // for one more entry. + bool Grow(InsertState& state); + + // Operational details of splitting a chain into two for Grow(). + void SplitForGrow(size_t grow_home, size_t old_home, int old_shift); + + // Takes an "under construction" entry and ensures it is no longer connected + // to its home chain (in preparaion for completing erasure and freeing the + // slot). Note that previous operations might have already noticed it being + // "under (de)construction" and removed it from its chain. + void Remove(HandleImpl* h); + + // Try to take ownership of an entry and erase+remove it from the table. + // Returns true if successful. Could fail if + // * There are other references to the entry + // * Some other thread has exclusive ownership or has freed it. + bool TryEraseHandle(HandleImpl* h, bool holding_ref, bool mark_invisible); + + // Calculates the appropriate maximum table size, for creating the memory + // mapping. + static size_t CalcMaxUsableLength( + size_t capacity, size_t min_avg_value_size, + CacheMetadataChargePolicy metadata_charge_policy); + + // Shared helper function that implements removing entries from a chain + // with proper handling to ensure all existing data is seen even in the + // presence of concurrent insertions, etc. (See implementation.) + template + void PurgeImpl(OpData* op_data, size_t home = SIZE_MAX, + EvictionData* data = nullptr); + + // An RAII wrapper for locking a chain of entries for removals. See + // implementation. + class ChainRewriteLock; + + // Helper function for PurgeImpl while holding a ChainRewriteLock. See + // implementation. + template + void PurgeImplLocked(OpData* op_data, ChainRewriteLock& rewrite_lock, + size_t home, EvictionData* data); + + // Update length_info_ as much as possible without waiting, given a known + // usable (ready for inserts and lookups) grow_home. (Previous grow_homes + // might not be usable yet, but we can check if they are by looking at + // the corresponding old home.) + void CatchUpLengthInfoNoWait(size_t known_usable_grow_home); + + private: // data + // mmaped area holding handles + const TypedMemMapping array_; + + // Metadata for table size under linear hashing. + // + // Lowest 8 bits are the minimum number of lowest hash bits to use + // ("min shift"). The upper 56 bits are a threshold. If that minumum number + // of bits taken from a hash value is < this threshold, then one more bit of + // hash value is taken and used. + // + // Other mechanisms (shift amounts on pointers) ensure complete availability + // of data already in the table even if a reader only sees a completely + // out-of-date version of this value. In the worst case, it could take + // log time to find the correct chain, but normally this value enables + // readers to find the correct chain on the first try. + // + // To maximize parallelization of Grow() operations, this field is only + // updated opportunistically after Grow() operations and in DoInsert() where + // it is found to be out-of-date. See CatchUpLengthInfoNoWait(). + AcqRelAtomic length_info_; + + // An already-computed version of the usable length times the max load + // factor. Could be slightly out of date but GrowIfNeeded()/Grow() handle + // that internally. + // (Relaxed: allowed to lag behind length_info_ by a little) + RelaxedAtomic occupancy_limit_; + + // The next index to use from array_ upon the next Grow(). Might be ahead of + // length_info_. + // (Relaxed: self-contained source of truth for next grow home) + RelaxedAtomic grow_frontier_; + + // See explanation in AutoHyperClockTable::Evict + // (Relaxed: allowed to lag behind clock_pointer_ and length_info_ state) + RelaxedAtomic clock_pointer_mask_; +}; // class AutoHyperClockTable // A single shard of sharded cache. -template +template class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase { public: + using Table = TableT; ClockCacheShard(size_t capacity, bool strict_capacity_limit, CacheMetadataChargePolicy metadata_charge_policy, MemoryAllocator* allocator, @@ -702,8 +1088,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase { return Lookup(key, hashed_key); } + Table& GetTable() { return table_; } + const Table& GetTable() const { return table_; } + #ifndef NDEBUG - size_t& TEST_MutableOccupancyLimit() const { + size_t& TEST_MutableOccupancyLimit() { return table_.TEST_MutableOccupancyLimit(); } // Acquire/release N references @@ -715,23 +1104,23 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase { Table table_; // Maximum total charge of all elements stored in the table. - std::atomic capacity_; + // (Relaxed: eventual consistency/update is OK) + RelaxedAtomic capacity_; - // Whether to reject insertion if cache reaches its full capacity. - std::atomic strict_capacity_limit_; + // Encodes eviction_effort_cap (bottom 31 bits) and strict_capacity_limit + // (top bit). See HyperClockCacheOptions::eviction_effort_cap etc. + // (Relaxed: eventual consistency/update is OK) + RelaxedAtomic eec_and_scl_; }; // class ClockCacheShard -class HyperClockCache -#ifdef NDEBUG - final -#endif - : public ShardedCache> { +template +class BaseHyperClockCache : public ShardedCache> { public: - using Shard = ClockCacheShard; - - explicit HyperClockCache(const HyperClockCacheOptions& opts); + using Shard = ClockCacheShard
; + using Handle = Cache::Handle; + using CacheItemHelper = Cache::CacheItemHelper; - const char* Name() const override { return "HyperClockCache"; } + explicit BaseHyperClockCache(const HyperClockCacheOptions& opts); Cache::ObjectPtr Value(Handle* handle) override; @@ -741,7 +1130,35 @@ class HyperClockCache void ReportProblems( const std::shared_ptr& /*info_log*/) const override; -}; // class HyperClockCache +}; + +class FixedHyperClockCache +#ifdef NDEBUG + final +#endif + : public BaseHyperClockCache { + public: + using BaseHyperClockCache::BaseHyperClockCache; + + const char* Name() const override { return "FixedHyperClockCache"; } + + void ReportProblems( + const std::shared_ptr& /*info_log*/) const override; +}; // class FixedHyperClockCache + +class AutoHyperClockCache +#ifdef NDEBUG + final +#endif + : public BaseHyperClockCache { + public: + using BaseHyperClockCache::BaseHyperClockCache; + + const char* Name() const override { return "AutoHyperClockCache"; } + + void ReportProblems( + const std::shared_ptr& /*info_log*/) const override; +}; // class AutoHyperClockCache } // namespace clock_cache diff --git a/cache/compressed_secondary_cache.cc b/cache/compressed_secondary_cache.cc index f5b7af308..6c19e7921 100644 --- a/cache/compressed_secondary_cache.cc +++ b/cache/compressed_secondary_cache.cc @@ -11,6 +11,7 @@ #include "memory/memory_allocator_impl.h" #include "monitoring/perf_context_imp.h" +#include "util/coding.h" #include "util/compression.h" #include "util/string_util.h" @@ -22,17 +23,24 @@ CompressedSecondaryCache::CompressedSecondaryCache( cache_options_(opts), cache_res_mgr_(std::make_shared( std::make_shared>( - cache_))) {} + cache_))), + disable_cache_(opts.capacity == 0) {} -CompressedSecondaryCache::~CompressedSecondaryCache() { - assert(cache_res_mgr_->GetTotalReservedCacheSize() == 0); -} +CompressedSecondaryCache::~CompressedSecondaryCache() {} std::unique_ptr CompressedSecondaryCache::Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase, - bool& kept_in_sec_cache) { + Statistics* stats, bool& kept_in_sec_cache) { assert(helper); + // This is a minor optimization. Its ok to skip it in TSAN in order to + // avoid a false positive. +#ifndef __SANITIZE_THREAD__ + if (disable_cache_) { + return nullptr; + } +#endif + std::unique_ptr handle; kept_in_sec_cache = false; Cache::Handle* lru_handle = cache_->Lookup(key); @@ -43,46 +51,72 @@ std::unique_ptr CompressedSecondaryCache::Lookup( void* handle_value = cache_->Value(lru_handle); if (handle_value == nullptr) { cache_->Release(lru_handle, /*erase_if_last_ref=*/false); + RecordTick(stats, COMPRESSED_SECONDARY_CACHE_DUMMY_HITS); return nullptr; } CacheAllocationPtr* ptr{nullptr}; CacheAllocationPtr merged_value; size_t handle_value_charge{0}; + const char* data_ptr = nullptr; + CacheTier source = CacheTier::kVolatileCompressedTier; + CompressionType type = cache_options_.compression_type; if (cache_options_.enable_custom_split_merge) { CacheValueChunk* value_chunk_ptr = reinterpret_cast(handle_value); merged_value = MergeChunksIntoValue(value_chunk_ptr, handle_value_charge); ptr = &merged_value; + data_ptr = ptr->get(); } else { + uint32_t type_32 = static_cast(type); + uint32_t source_32 = static_cast(source); ptr = reinterpret_cast(handle_value); handle_value_charge = cache_->GetCharge(lru_handle); + data_ptr = ptr->get(); + data_ptr = GetVarint32Ptr(data_ptr, data_ptr + 1, + static_cast(&type_32)); + type = static_cast(type_32); + data_ptr = GetVarint32Ptr(data_ptr, data_ptr + 1, + static_cast(&source_32)); + source = static_cast(source_32); + handle_value_charge -= (data_ptr - ptr->get()); } MemoryAllocator* allocator = cache_options_.memory_allocator.get(); Status s; Cache::ObjectPtr value{nullptr}; size_t charge{0}; - if (cache_options_.compression_type == kNoCompression || - cache_options_.do_not_compress_roles.Contains(helper->role)) { - s = helper->create_cb(Slice(ptr->get(), handle_value_charge), - create_context, allocator, &value, &charge); - } else { - UncompressionContext uncompression_context(cache_options_.compression_type); - UncompressionInfo uncompression_info(uncompression_context, - UncompressionDict::GetEmptyDict(), - cache_options_.compression_type); - - size_t uncompressed_size{0}; - CacheAllocationPtr uncompressed = UncompressData( - uncompression_info, (char*)ptr->get(), handle_value_charge, - &uncompressed_size, cache_options_.compress_format_version, allocator); - - if (!uncompressed) { - cache_->Release(lru_handle, /*erase_if_last_ref=*/true); - return nullptr; + if (source == CacheTier::kVolatileCompressedTier) { + if (cache_options_.compression_type == kNoCompression || + cache_options_.do_not_compress_roles.Contains(helper->role)) { + s = helper->create_cb(Slice(data_ptr, handle_value_charge), + kNoCompression, CacheTier::kVolatileTier, + create_context, allocator, &value, &charge); + } else { + UncompressionContext uncompression_context( + cache_options_.compression_type); + UncompressionInfo uncompression_info(uncompression_context, + UncompressionDict::GetEmptyDict(), + cache_options_.compression_type); + + size_t uncompressed_size{0}; + CacheAllocationPtr uncompressed = + UncompressData(uncompression_info, (char*)data_ptr, + handle_value_charge, &uncompressed_size, + cache_options_.compress_format_version, allocator); + + if (!uncompressed) { + cache_->Release(lru_handle, /*erase_if_last_ref=*/true); + return nullptr; + } + s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size), + kNoCompression, CacheTier::kVolatileTier, + create_context, allocator, &value, &charge); } - s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size), + } else { + // The item was not compressed by us. Let the helper create_cb + // uncompress it + s = helper->create_cb(Slice(data_ptr, handle_value_charge), type, source, create_context, allocator, &value, &charge); } @@ -104,43 +138,63 @@ std::unique_ptr CompressedSecondaryCache::Lookup( cache_->Release(lru_handle, /*erase_if_last_ref=*/false); } handle.reset(new CompressedSecondaryCacheResultHandle(value, charge)); + RecordTick(stats, COMPRESSED_SECONDARY_CACHE_HITS); return handle; } -Status CompressedSecondaryCache::Insert(const Slice& key, - Cache::ObjectPtr value, - const Cache::CacheItemHelper* helper) { - if (value == nullptr) { - return Status::InvalidArgument(); - } - - Cache::Handle* lru_handle = cache_->Lookup(key); +bool CompressedSecondaryCache::MaybeInsertDummy(const Slice& key) { auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge); + Cache::Handle* lru_handle = cache_->Lookup(key); if (lru_handle == nullptr) { PERF_COUNTER_ADD(compressed_sec_cache_insert_dummy_count, 1); // Insert a dummy handle if the handle is evicted for the first time. - return cache_->Insert(key, /*obj=*/nullptr, internal_helper, - /*charge=*/0); + cache_->Insert(key, /*obj=*/nullptr, internal_helper, /*charge=*/0) + .PermitUncheckedError(); + return true; } else { cache_->Release(lru_handle, /*erase_if_last_ref=*/false); } - size_t size = (*helper->size_cb)(value); + return false; +} + +Status CompressedSecondaryCache::InsertInternal( + const Slice& key, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, CompressionType type, + CacheTier source) { + if (source != CacheTier::kVolatileCompressedTier && + cache_options_.enable_custom_split_merge) { + // We don't support custom split/merge for the tiered case + return Status::OK(); + } + + auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge); + char header[10]; + char* payload = header; + payload = EncodeVarint32(payload, static_cast(type)); + payload = EncodeVarint32(payload, static_cast(source)); + + size_t header_size = payload - header; + size_t data_size = (*helper->size_cb)(value); + size_t total_size = data_size + header_size; CacheAllocationPtr ptr = - AllocateBlock(size, cache_options_.memory_allocator.get()); + AllocateBlock(total_size, cache_options_.memory_allocator.get()); + char* data_ptr = ptr.get() + header_size; - Status s = (*helper->saveto_cb)(value, 0, size, ptr.get()); + Status s = (*helper->saveto_cb)(value, 0, data_size, data_ptr); if (!s.ok()) { return s; } - Slice val(ptr.get(), size); + Slice val(data_ptr, data_size); std::string compressed_val; if (cache_options_.compression_type != kNoCompression && + type == kNoCompression && !cache_options_.do_not_compress_roles.Contains(helper->role)) { - PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, size); + PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, data_size); CompressionOptions compression_opts; - CompressionContext compression_context(cache_options_.compression_type); + CompressionContext compression_context(cache_options_.compression_type, + compression_opts); uint64_t sample_for_compression{0}; CompressionInfo compression_info( compression_opts, compression_context, CompressionDict::GetEmptyDict(), @@ -155,12 +209,14 @@ Status CompressedSecondaryCache::Insert(const Slice& key, } val = Slice(compressed_val); - size = compressed_val.size(); - PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, size); + data_size = compressed_val.size(); + total_size = header_size + data_size; + PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, data_size); if (!cache_options_.enable_custom_split_merge) { - ptr = AllocateBlock(size, cache_options_.memory_allocator.get()); - memcpy(ptr.get(), compressed_val.data(), size); + ptr = AllocateBlock(total_size, cache_options_.memory_allocator.get()); + data_ptr = ptr.get() + header_size; + memcpy(data_ptr, compressed_val.data(), data_size); } } @@ -171,9 +227,43 @@ Status CompressedSecondaryCache::Insert(const Slice& key, SplitValueIntoChunks(val, cache_options_.compression_type, charge); return cache_->Insert(key, value_chunks_head, internal_helper, charge); } else { + std::memcpy(ptr.get(), header, header_size); CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr)); - return cache_->Insert(key, buf, internal_helper, size); + return cache_->Insert(key, buf, internal_helper, total_size); + } +} + +Status CompressedSecondaryCache::Insert(const Slice& key, + Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, + bool force_insert) { + if (value == nullptr) { + return Status::InvalidArgument(); } + + if (!force_insert && MaybeInsertDummy(key)) { + return Status::OK(); + } + + return InsertInternal(key, value, helper, kNoCompression, + CacheTier::kVolatileCompressedTier); +} + +Status CompressedSecondaryCache::InsertSaved( + const Slice& key, const Slice& saved, CompressionType type = kNoCompression, + CacheTier source = CacheTier::kVolatileTier) { + if (type == kNoCompression) { + return Status::OK(); + } + + auto slice_helper = &kSliceCacheItemHelper; + if (MaybeInsertDummy(key)) { + return Status::OK(); + } + + return InsertInternal( + key, static_cast(const_cast(&saved)), + slice_helper, type, source); } void CompressedSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); } @@ -182,6 +272,7 @@ Status CompressedSecondaryCache::SetCapacity(size_t capacity) { MutexLock l(&capacity_mutex_); cache_options_.capacity = capacity; cache_->SetCapacity(capacity); + disable_cache_ = capacity == 0; return Status::OK(); } diff --git a/cache/compressed_secondary_cache.h b/cache/compressed_secondary_cache.h index 7bee05955..90e134fcf 100644 --- a/cache/compressed_secondary_cache.h +++ b/cache/compressed_secondary_cache.h @@ -77,12 +77,16 @@ class CompressedSecondaryCache : public SecondaryCache { const char* Name() const override { return "CompressedSecondaryCache"; } Status Insert(const Slice& key, Cache::ObjectPtr value, - const Cache::CacheItemHelper* helper) override; + const Cache::CacheItemHelper* helper, + bool force_insert) override; + + Status InsertSaved(const Slice& key, const Slice& saved, CompressionType type, + CacheTier source) override; std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase, - bool& kept_in_sec_cache) override; + Statistics* stats, bool& kept_in_sec_cache) override; bool SupportForceErase() const override { return true; } @@ -129,12 +133,19 @@ class CompressedSecondaryCache : public SecondaryCache { CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head, size_t& charge); + bool MaybeInsertDummy(const Slice& key); + + Status InsertInternal(const Slice& key, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, + CompressionType type, CacheTier source); + // TODO: clean up to use cleaner interfaces in typed_cache.h const Cache::CacheItemHelper* GetHelper(bool enable_custom_split_merge) const; std::shared_ptr cache_; CompressedSecondaryCacheOptions cache_options_; mutable port::Mutex capacity_mutex_; std::shared_ptr cache_res_mgr_; + bool disable_cache_; }; } // namespace ROCKSDB_NAMESPACE diff --git a/cache/compressed_secondary_cache_test.cc b/cache/compressed_secondary_cache_test.cc index 4e6a1375b..ac1786f16 100644 --- a/cache/compressed_secondary_cache_test.cc +++ b/cache/compressed_secondary_cache_test.cc @@ -12,6 +12,7 @@ #include "cache/secondary_cache_adapter.h" #include "memory/jemalloc_nodump_allocator.h" +#include "rocksdb/cache.h" #include "rocksdb/convenience.h" #include "test_util/secondary_cache_test_util.h" #include "test_util/testharness.h" @@ -43,7 +44,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, // Lookup an non-existent key. std::unique_ptr handle0 = sec_cache->Lookup(key0, GetHelper(), this, true, /*advise_erase=*/true, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle0, nullptr); Random rnd(301); @@ -51,23 +52,23 @@ class CompressedSecondaryCacheTestBase : public testing::Test, std::string str1(rnd.RandomString(1000)); TestItem item1(str1.data(), str1.length()); // A dummy handle is inserted if the item is inserted for the first time. - ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1); ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); std::unique_ptr handle1_1 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle1_1, nullptr); // Insert and Lookup the item k1 for the second time and advise erasing it. - ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1); std::unique_ptr handle1_2 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_NE(handle1_2, nullptr); ASSERT_FALSE(kept_in_sec_cache); if (sec_cache_is_compressed) { @@ -88,20 +89,20 @@ class CompressedSecondaryCacheTestBase : public testing::Test, // Lookup the item k1 again. std::unique_ptr handle1_3 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle1_3, nullptr); // Insert and Lookup the item k2. std::string str2(rnd.RandomString(1000)); TestItem item2(str2.data(), str2.length()); - ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper())); + ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2); std::unique_ptr handle2_1 = sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle2_1, nullptr); - ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper())); + ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2); if (sec_cache_is_compressed) { ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, @@ -114,7 +115,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, } std::unique_ptr handle2_2 = sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_NE(handle2_2, nullptr); std::unique_ptr val2 = std::unique_ptr(static_cast(handle2_2->Value())); @@ -183,26 +184,26 @@ class CompressedSecondaryCacheTestBase : public testing::Test, std::string str1(rnd.RandomString(1000)); TestItem item1(str1.data(), str1.length()); // Insert a dummy handle. - ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false)); // Insert k1. - ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false)); // Insert and Lookup the second item. std::string str2(rnd.RandomString(200)); TestItem item2(str2.data(), str2.length()); // Insert a dummy handle, k1 is not evicted. - ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper())); + ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false)); bool kept_in_sec_cache{false}; std::unique_ptr handle1 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle1, nullptr); // Insert k2 and k1 is evicted. - ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper())); + ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false)); std::unique_ptr handle2 = sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_NE(handle2, nullptr); std::unique_ptr val2 = std::unique_ptr(static_cast(handle2->Value())); @@ -210,26 +211,26 @@ class CompressedSecondaryCacheTestBase : public testing::Test, ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0); // Insert k1 again and a dummy handle is inserted. - ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false)); std::unique_ptr handle1_1 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle1_1, nullptr); // Create Fails. SetFailCreate(true); std::unique_ptr handle2_1 = sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/true, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle2_1, nullptr); // Save Fails. std::string str3 = rnd.RandomString(10); TestItem item3(str3.data(), str3.length()); // The Status is OK because a dummy handle is inserted. - ASSERT_OK(sec_cache->Insert(key3, &item3, GetHelperFail())); - ASSERT_NOK(sec_cache->Insert(key3, &item3, GetHelperFail())); + ASSERT_OK(sec_cache->Insert(key3, &item3, GetHelperFail(), false)); + ASSERT_NOK(sec_cache->Insert(key3, &item3, GetHelperFail(), false)); sec_cache.reset(); } @@ -737,7 +738,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, class CompressedSecondaryCacheTest : public CompressedSecondaryCacheTestBase, public testing::WithParamInterface { - const std::string& Type() override { return GetParam(); } + const std::string& Type() const override { return GetParam(); } }; INSTANTIATE_TEST_CASE_P(CompressedSecondaryCacheTest, @@ -752,7 +753,7 @@ class CompressedSecCacheTestWithCompressAndAllocatorParam sec_cache_is_compressed_ = std::get<0>(GetParam()); use_jemalloc_ = std::get<1>(GetParam()); } - const std::string& Type() override { return std::get<2>(GetParam()); } + const std::string& Type() const override { return std::get<2>(GetParam()); } bool sec_cache_is_compressed_; bool use_jemalloc_; }; @@ -773,7 +774,7 @@ class CompressedSecondaryCacheTestWithCompressionParam CompressedSecondaryCacheTestWithCompressionParam() { sec_cache_is_compressed_ = std::get<0>(GetParam()); } - const std::string& Type() override { return std::get<1>(GetParam()); } + const std::string& Type() const override { return std::get<1>(GetParam()); } bool sec_cache_is_compressed_; }; @@ -904,16 +905,16 @@ TEST_P(CompressedSecondaryCacheTestWithCompressionParam, EntryRoles) { Slice ith_key = Slice(junk.data(), 16); get_perf_context()->Reset(); - ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role))); + ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role), false)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1U); - ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role))); + ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role), false)); ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1U); bool kept_in_sec_cache{true}; - std::unique_ptr handle = - sec_cache->Lookup(ith_key, GetHelper(role), this, true, - /*advise_erase=*/true, kept_in_sec_cache); + std::unique_ptr handle = sec_cache->Lookup( + ith_key, GetHelper(role), this, true, + /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache); ASSERT_NE(handle, nullptr); // Lookup returns the right data @@ -950,7 +951,7 @@ class CompressedSecCacheTestWithCompressAndSplitParam sec_cache_is_compressed_ = std::get<0>(GetParam()); enable_custom_split_merge_ = std::get<1>(GetParam()); } - const std::string& Type() override { return std::get<2>(GetParam()); } + const std::string& Type() const override { return std::get<2>(GetParam()); } bool sec_cache_is_compressed_; bool enable_custom_split_merge_; }; @@ -976,24 +977,58 @@ TEST_P(CompressedSecondaryCacheTest, SplictValueAndMergeChunksTest) { SplictValueAndMergeChunksTest(); } -class CompressedSecCacheTestWithTiered : public ::testing::Test { +using secondary_cache_test_util::WithCacheType; + +class CompressedSecCacheTestWithTiered + : public testing::Test, + public WithCacheType, + public testing::WithParamInterface< + std::tuple> { public: + using secondary_cache_test_util::WithCacheType::TestItem; CompressedSecCacheTestWithTiered() { LRUCacheOptions lru_opts; - TieredVolatileCacheOptions opts; - lru_opts.capacity = 70 << 20; - opts.cache_opts = &lru_opts; - opts.cache_type = PrimaryCacheType::kCacheTypeLRU; - opts.comp_cache_opts.capacity = 30 << 20; - cache_ = NewTieredVolatileCache(opts); + HyperClockCacheOptions hcc_opts( + /*_capacity=*/0, + /*_estimated_entry_charge=*/256 << 10, + /*_num_shard_bits=*/0); + // eviction_effort_cap setting simply to avoid churn in existing test + hcc_opts.eviction_effort_cap = 100; + TieredCacheOptions opts; + lru_opts.capacity = 0; + lru_opts.num_shard_bits = 0; + lru_opts.high_pri_pool_ratio = 0; + opts.cache_type = std::get<0>(GetParam()); + if (opts.cache_type == PrimaryCacheType::kCacheTypeLRU) { + opts.cache_opts = &lru_opts; + } else { + opts.cache_opts = &hcc_opts; + } + opts.adm_policy = std::get<1>(GetParam()); + ; + opts.comp_cache_opts.capacity = 0; + opts.comp_cache_opts.num_shard_bits = 0; + opts.total_capacity = 100 << 20; + opts.compressed_secondary_ratio = 0.3; + cache_ = NewTieredCache(opts); cache_res_mgr_ = std::make_shared>( cache_); } + const std::string& Type() const override { + if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) { + return lru_str; + } else { + return hcc_str; + } + } + protected: CacheReservationManager* cache_res_mgr() { return cache_res_mgr_.get(); } + std::shared_ptr GetTieredCache() { return cache_; } + Cache* GetCache() { return static_cast_with_check( cache_.get()) @@ -1013,13 +1048,19 @@ class CompressedSecCacheTestWithTiered : public ::testing::Test { private: std::shared_ptr cache_; std::shared_ptr cache_res_mgr_; + static std::string lru_str; + static std::string hcc_str; }; +std::string CompressedSecCacheTestWithTiered::lru_str(WithCacheType::kLRU); +std::string CompressedSecCacheTestWithTiered::hcc_str( + WithCacheType::kFixedHyperClock); + bool CacheUsageWithinBounds(size_t val1, size_t val2, size_t error) { return ((val1 < (val2 + error)) && (val1 > (val2 - error))); } -TEST_F(CompressedSecCacheTestWithTiered, CacheReservationManager) { +TEST_P(CompressedSecCacheTestWithTiered, CacheReservationManager) { CompressedSecondaryCache* sec_cache = reinterpret_cast(GetSecondaryCache()); @@ -1041,7 +1082,7 @@ TEST_F(CompressedSecCacheTestWithTiered, CacheReservationManager) { EXPECT_EQ(sec_cache->TEST_GetUsage(), 0); } -TEST_F(CompressedSecCacheTestWithTiered, +TEST_P(CompressedSecCacheTestWithTiered, CacheReservationManagerMultipleUpdate) { CompressedSecondaryCache* sec_cache = reinterpret_cast(GetSecondaryCache()); @@ -1067,6 +1108,268 @@ TEST_F(CompressedSecCacheTestWithTiered, EXPECT_EQ(sec_cache->TEST_GetUsage(), 0); } +TEST_P(CompressedSecCacheTestWithTiered, AdmissionPolicy) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_BYPASS("This test requires LZ4 support\n"); + return; + } + + Cache* tiered_cache = GetTieredCache().get(); + Cache* cache = GetCache(); + std::vector keys; + std::vector vals; + // Make the item size slightly less than 10MB to ensure we can fit the + // expected number of items in the cache + int item_size = (10 << 20) - (1 << 18); + int i; + Random rnd(301); + for (i = 0; i < 14; ++i) { + keys.emplace_back(CacheKey::CreateUniqueForCacheLifetime(cache)); + vals.emplace_back(rnd.RandomString(item_size)); + } + + for (i = 0; i < 7; ++i) { + TestItem* item = new TestItem(vals[i].data(), vals[i].length()); + ASSERT_OK(tiered_cache->Insert(keys[i].AsSlice(), item, GetHelper(), + vals[i].length())); + } + + Cache::Handle* handle1; + handle1 = tiered_cache->Lookup(keys[0].AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle1, nullptr); + Cache::Handle* handle2; + handle2 = tiered_cache->Lookup(keys[1].AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle2, nullptr); + tiered_cache->Release(handle1); + tiered_cache->Release(handle2); + + // Flush all previous entries out of the primary cache + for (i = 7; i < 14; ++i) { + TestItem* item = new TestItem(vals[i].data(), vals[i].length()); + ASSERT_OK(tiered_cache->Insert(keys[i].AsSlice(), item, GetHelper(), + vals[i].length())); + } + // keys 0 and 1 should be found as they had the hit bit set + handle1 = tiered_cache->Lookup(keys[0].AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle1, nullptr); + handle2 = tiered_cache->Lookup(keys[1].AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle2, nullptr); + tiered_cache->Release(handle1); + tiered_cache->Release(handle2); + + handle1 = tiered_cache->Lookup(keys[2].AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_EQ(handle1, nullptr); + handle1 = tiered_cache->Lookup(keys[3].AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_EQ(handle1, nullptr); +} + +TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdate) { + CompressedSecondaryCache* sec_cache = + reinterpret_cast(GetSecondaryCache()); + std::shared_ptr tiered_cache = GetTieredCache(); + + // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to + // double explicit casts + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20), + GetPercent(30 << 20, 1)); + size_t sec_capacity; + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (30 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, 130 << 20)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (39 << 20), + GetPercent(39 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (39 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, 70 << 20)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (21 << 20), + GetPercent(21 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (21 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20), + GetPercent(30 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (30 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.4)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (40 << 20), + GetPercent(40 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (40 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.2)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (20 << 20), + GetPercent(20 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (20 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 1.0)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (100 << 20), + GetPercent(100 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, 100 << 20); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.0)); + // Only check usage for LRU cache. HCC shows a 64KB usage for some reason + if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) { + ASSERT_EQ(GetCache()->GetUsage(), 0); + } + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, 0); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.3)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20), + GetPercent(30 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (30 << 20)); +} + +TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdateWithReservation) { + CompressedSecondaryCache* sec_cache = + reinterpret_cast(GetSecondaryCache()); + std::shared_ptr tiered_cache = GetTieredCache(); + + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(10 << 20)); + // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to + // double explicit casts + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20), + GetPercent(37 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20), + GetPercent(3 << 20, 1)); + size_t sec_capacity; + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (30 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, 70 << 20)); + // Only check usage for LRU cache. HCC is slightly off for some reason + if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) { + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (28 << 20), + GetPercent(28 << 20, 1)); + } + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20), + GetPercent(3 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (21 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, 130 << 20)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (46 << 20), + GetPercent(46 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20), + GetPercent(3 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (39 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20), + GetPercent(37 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20), + GetPercent(3 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (30 << 20)); + + ASSERT_OK(tiered_cache->GetSecondaryCacheCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, 30 << 20); + size_t sec_usage; + ASSERT_OK(tiered_cache->GetSecondaryCachePinnedUsage(sec_usage)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_usage, 3 << 20, + GetPercent(3 << 20, 1)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.39)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (45 << 20), + GetPercent(45 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (4 << 20), + GetPercent(4 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (39 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.2)); + // Only check usage for LRU cache. HCC is slightly off for some reason + if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) { + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (28 << 20), + GetPercent(28 << 20, 1)); + } + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (2 << 20), + GetPercent(2 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (20 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 1.0)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (100 << 20), + GetPercent(100 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (10 << 20), + GetPercent(10 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, 100 << 20); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.0)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (10 << 20), + GetPercent(10 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, 0); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.3)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20), + GetPercent(37 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20), + GetPercent(3 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, 30 << 20); + + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0)); +} + +TEST_P(CompressedSecCacheTestWithTiered, ReservationOverCapacity) { + CompressedSecondaryCache* sec_cache = + reinterpret_cast(GetSecondaryCache()); + std::shared_ptr tiered_cache = GetTieredCache(); + + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(110 << 20)); + // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to + // double explicit casts + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (110 << 20), + GetPercent(110 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (30 << 20), + GetPercent(30 << 20, 1)); + size_t sec_capacity; + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (30 << 20)); + + ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.39)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (110 << 20), + GetPercent(110 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (39 << 20), + GetPercent(39 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (39 << 20)); + + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(90 << 20)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (94 << 20), + GetPercent(94 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (35 << 20), + GetPercent(35 << 20, 1)); + ASSERT_OK(sec_cache->GetCapacity(sec_capacity)); + ASSERT_EQ(sec_capacity, (39 << 20)); + + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0)); +} + +INSTANTIATE_TEST_CASE_P( + CompressedSecCacheTests, CompressedSecCacheTestWithTiered, + ::testing::Values( + std::make_tuple(PrimaryCacheType::kCacheTypeLRU, + TieredAdmissionPolicy::kAdmPolicyAllowCacheHits), + std::make_tuple(PrimaryCacheType::kCacheTypeHCC, + TieredAdmissionPolicy::kAdmPolicyAllowCacheHits))); + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/cache/lru_cache.cc b/cache/lru_cache.cc index b72c82403..9d1695224 100644 --- a/cache/lru_cache.cc +++ b/cache/lru_cache.cc @@ -340,7 +340,8 @@ void LRUCacheShard::NotifyEvicted( for (LRUHandle* entry : evicted_handles) { if (eviction_callback_ && eviction_callback_(entry->key(), - reinterpret_cast(entry))) { + reinterpret_cast(entry), + entry->HasHit())) { // Callback took ownership of obj; just free handle free(entry); } else { @@ -505,7 +506,8 @@ bool LRUCacheShard::Release(LRUHandle* e, bool /*useful*/, // Only call eviction callback if we're sure no one requested erasure // FIXME: disabled because of test churn if (false && was_in_cache && !erase_if_last_ref && eviction_callback_ && - eviction_callback_(e->key(), reinterpret_cast(e))) { + eviction_callback_(e->key(), reinterpret_cast(e), + e->HasHit())) { // Callback took ownership of obj; just free handle free(e); } else { diff --git a/cache/lru_cache_test.cc b/cache/lru_cache_test.cc index cb7beb7b1..9f70a54cf 100644 --- a/cache/lru_cache_test.cc +++ b/cache/lru_cache_test.cc @@ -371,11 +371,12 @@ TEST_F(LRUCacheTest, EntriesWithPriority) { namespace clock_cache { +template class ClockCacheTest : public testing::Test { public: - using Shard = HyperClockCache::Shard; - using Table = HyperClockTable; - using HandleImpl = Shard::HandleImpl; + using Shard = typename ClockCache::Shard; + using Table = typename Shard::Table; + using TableOpts = typename Table::Opts; ClockCacheTest() {} ~ClockCacheTest() override { DeleteShard(); } @@ -388,13 +389,13 @@ class ClockCacheTest : public testing::Test { } } - void NewShard(size_t capacity, bool strict_capacity_limit = true) { + void NewShard(size_t capacity, bool strict_capacity_limit = true, + int eviction_effort_cap = 30) { DeleteShard(); shard_ = reinterpret_cast(port::cacheline_aligned_alloc(sizeof(Shard))); - Table::Opts opts; - opts.estimated_value_size = 1; + TableOpts opts{1 /*value_size*/, eviction_effort_cap}; new (shard_) Shard(capacity, strict_capacity_limit, kDontChargeCacheMetadata, /*allocator*/ nullptr, &eviction_callback_, &hash_seed_, opts); @@ -445,12 +446,20 @@ class ClockCacheTest : public testing::Test { return Slice(reinterpret_cast(&hashed_key), 16U); } + // A bad hash function for testing / stressing collision handling static inline UniqueId64x2 TestHashedKey(char key) { // For testing hash near-collision behavior, put the variance in // hashed_key in bits that are unlikely to be used as hash bits. return {(static_cast(key) << 56) + 1234U, 5678U}; } + // A reasonable hash function, for testing "typical behavior" etc. + template + static inline UniqueId64x2 CheapHash(T i) { + return {static_cast(i) * uint64_t{0x85EBCA77C2B2AE63}, + static_cast(i) * uint64_t{0xC2B2AE3D27D4EB4F}}; + } + Shard* shard_ = nullptr; private: @@ -458,43 +467,53 @@ class ClockCacheTest : public testing::Test { uint32_t hash_seed_ = 0; }; -TEST_F(ClockCacheTest, Misc) { - NewShard(3); +using ClockCacheTypes = + ::testing::Types; +TYPED_TEST_CASE(ClockCacheTest, ClockCacheTypes); + +TYPED_TEST(ClockCacheTest, Misc) { + this->NewShard(3); + // NOTE: templated base class prevents simple naming of inherited members, + // so lots of `this->` + auto& shard = *this->shard_; // Key size stuff - EXPECT_OK(InsertWithLen('a', 16)); - EXPECT_NOK(InsertWithLen('b', 15)); - EXPECT_OK(InsertWithLen('b', 16)); - EXPECT_NOK(InsertWithLen('c', 17)); - EXPECT_NOK(InsertWithLen('d', 1000)); - EXPECT_NOK(InsertWithLen('e', 11)); - EXPECT_NOK(InsertWithLen('f', 0)); + EXPECT_OK(this->InsertWithLen('a', 16)); + EXPECT_NOK(this->InsertWithLen('b', 15)); + EXPECT_OK(this->InsertWithLen('b', 16)); + EXPECT_NOK(this->InsertWithLen('c', 17)); + EXPECT_NOK(this->InsertWithLen('d', 1000)); + EXPECT_NOK(this->InsertWithLen('e', 11)); + EXPECT_NOK(this->InsertWithLen('f', 0)); // Some of this is motivated by code coverage std::string wrong_size_key(15, 'x'); - EXPECT_FALSE(Lookup(wrong_size_key, TestHashedKey('x'))); - EXPECT_FALSE(shard_->Ref(nullptr)); - EXPECT_FALSE(shard_->Release(nullptr)); - shard_->Erase(wrong_size_key, TestHashedKey('x')); // no-op + EXPECT_FALSE(this->Lookup(wrong_size_key, this->TestHashedKey('x'))); + EXPECT_FALSE(shard.Ref(nullptr)); + EXPECT_FALSE(shard.Release(nullptr)); + shard.Erase(wrong_size_key, this->TestHashedKey('x')); // no-op } -TEST_F(ClockCacheTest, Limits) { - constexpr size_t kCapacity = 3; - NewShard(kCapacity, false /*strict_capacity_limit*/); +TYPED_TEST(ClockCacheTest, Limits) { + constexpr size_t kCapacity = 64; + this->NewShard(kCapacity, false /*strict_capacity_limit*/); + auto& shard = *this->shard_; + using HandleImpl = typename ClockCacheTest::Shard::HandleImpl; + for (bool strict_capacity_limit : {false, true, false}) { SCOPED_TRACE("strict_capacity_limit = " + std::to_string(strict_capacity_limit)); // Also tests switching between strict limit and not - shard_->SetStrictCapacityLimit(strict_capacity_limit); + shard.SetStrictCapacityLimit(strict_capacity_limit); - UniqueId64x2 hkey = TestHashedKey('x'); + UniqueId64x2 hkey = this->TestHashedKey('x'); // Single entry charge beyond capacity { - Status s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, - &kNoopCacheItemHelper, 5 /*charge*/, - nullptr /*handle*/, Cache::Priority::LOW); + Status s = shard.Insert(this->TestKey(hkey), hkey, nullptr /*value*/, + &kNoopCacheItemHelper, kCapacity + 2 /*charge*/, + nullptr /*handle*/, Cache::Priority::LOW); if (strict_capacity_limit) { EXPECT_TRUE(s.IsMemoryLimit()); } else { @@ -505,11 +524,11 @@ TEST_F(ClockCacheTest, Limits) { // Single entry fills capacity { HandleImpl* h; - ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, - &kNoopCacheItemHelper, 3 /*charge*/, &h, - Cache::Priority::LOW)); + ASSERT_OK(shard.Insert(this->TestKey(hkey), hkey, nullptr /*value*/, + &kNoopCacheItemHelper, kCapacity /*charge*/, &h, + Cache::Priority::LOW)); // Try to insert more - Status s = Insert('a'); + Status s = this->Insert('a'); if (strict_capacity_limit) { EXPECT_TRUE(s.IsMemoryLimit()); } else { @@ -517,22 +536,22 @@ TEST_F(ClockCacheTest, Limits) { } // Release entry filling capacity. // Cover useful = false case. - shard_->Release(h, false /*useful*/, false /*erase_if_last_ref*/); + shard.Release(h, false /*useful*/, false /*erase_if_last_ref*/); } // Insert more than table size can handle to exceed occupancy limit. // (Cleverly using mostly zero-charge entries, but some non-zero to // verify usage tracking on detached entries.) { - size_t n = shard_->GetTableAddressCount() + 1; + size_t n = kCapacity * 5 + 1; std::unique_ptr ha { new HandleImpl* [n] {} }; Status s; for (size_t i = 0; i < n && s.ok(); ++i) { hkey[1] = i; - s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, - &kNoopCacheItemHelper, - (i + kCapacity < n) ? 0 : 1 /*charge*/, &ha[i], - Cache::Priority::LOW); + s = shard.Insert(this->TestKey(hkey), hkey, nullptr /*value*/, + &kNoopCacheItemHelper, + (i + kCapacity < n) ? 0 : 1 /*charge*/, &ha[i], + Cache::Priority::LOW); if (i == 0) { EXPECT_OK(s); } @@ -543,130 +562,180 @@ TEST_F(ClockCacheTest, Limits) { EXPECT_OK(s); } // Same result if not keeping a reference - s = Insert('a'); + s = this->Insert('a'); if (strict_capacity_limit) { EXPECT_TRUE(s.IsMemoryLimit()); } else { EXPECT_OK(s); } + EXPECT_EQ(shard.GetOccupancyCount(), shard.GetOccupancyLimit()); + // Regardless, we didn't allow table to actually get full - EXPECT_LT(shard_->GetOccupancyCount(), shard_->GetTableAddressCount()); + EXPECT_LT(shard.GetOccupancyCount(), shard.GetTableAddressCount()); // Release handles for (size_t i = 0; i < n; ++i) { if (ha[i]) { - shard_->Release(ha[i]); + shard.Release(ha[i]); } } } } } -TEST_F(ClockCacheTest, ClockEvictionTest) { +TYPED_TEST(ClockCacheTest, ClockEvictionTest) { for (bool strict_capacity_limit : {false, true}) { SCOPED_TRACE("strict_capacity_limit = " + std::to_string(strict_capacity_limit)); - NewShard(6, strict_capacity_limit); - EXPECT_OK(Insert('a', Cache::Priority::BOTTOM)); - EXPECT_OK(Insert('b', Cache::Priority::LOW)); - EXPECT_OK(Insert('c', Cache::Priority::HIGH)); - EXPECT_OK(Insert('d', Cache::Priority::BOTTOM)); - EXPECT_OK(Insert('e', Cache::Priority::LOW)); - EXPECT_OK(Insert('f', Cache::Priority::HIGH)); - - EXPECT_TRUE(Lookup('a', /*use*/ false)); - EXPECT_TRUE(Lookup('b', /*use*/ false)); - EXPECT_TRUE(Lookup('c', /*use*/ false)); - EXPECT_TRUE(Lookup('d', /*use*/ false)); - EXPECT_TRUE(Lookup('e', /*use*/ false)); - EXPECT_TRUE(Lookup('f', /*use*/ false)); + this->NewShard(6, strict_capacity_limit); + auto& shard = *this->shard_; + EXPECT_OK(this->Insert('a', Cache::Priority::BOTTOM)); + EXPECT_OK(this->Insert('b', Cache::Priority::LOW)); + EXPECT_OK(this->Insert('c', Cache::Priority::HIGH)); + EXPECT_OK(this->Insert('d', Cache::Priority::BOTTOM)); + EXPECT_OK(this->Insert('e', Cache::Priority::LOW)); + EXPECT_OK(this->Insert('f', Cache::Priority::HIGH)); + + EXPECT_TRUE(this->Lookup('a', /*use*/ false)); + EXPECT_TRUE(this->Lookup('b', /*use*/ false)); + EXPECT_TRUE(this->Lookup('c', /*use*/ false)); + EXPECT_TRUE(this->Lookup('d', /*use*/ false)); + EXPECT_TRUE(this->Lookup('e', /*use*/ false)); + EXPECT_TRUE(this->Lookup('f', /*use*/ false)); // Ensure bottom are evicted first, even if new entries are low - EXPECT_OK(Insert('g', Cache::Priority::LOW)); - EXPECT_OK(Insert('h', Cache::Priority::LOW)); - - EXPECT_FALSE(Lookup('a', /*use*/ false)); - EXPECT_TRUE(Lookup('b', /*use*/ false)); - EXPECT_TRUE(Lookup('c', /*use*/ false)); - EXPECT_FALSE(Lookup('d', /*use*/ false)); - EXPECT_TRUE(Lookup('e', /*use*/ false)); - EXPECT_TRUE(Lookup('f', /*use*/ false)); + EXPECT_OK(this->Insert('g', Cache::Priority::LOW)); + EXPECT_OK(this->Insert('h', Cache::Priority::LOW)); + + EXPECT_FALSE(this->Lookup('a', /*use*/ false)); + EXPECT_TRUE(this->Lookup('b', /*use*/ false)); + EXPECT_TRUE(this->Lookup('c', /*use*/ false)); + EXPECT_FALSE(this->Lookup('d', /*use*/ false)); + EXPECT_TRUE(this->Lookup('e', /*use*/ false)); + EXPECT_TRUE(this->Lookup('f', /*use*/ false)); // Mark g & h useful - EXPECT_TRUE(Lookup('g', /*use*/ true)); - EXPECT_TRUE(Lookup('h', /*use*/ true)); + EXPECT_TRUE(this->Lookup('g', /*use*/ true)); + EXPECT_TRUE(this->Lookup('h', /*use*/ true)); // Then old LOW entries - EXPECT_OK(Insert('i', Cache::Priority::LOW)); - EXPECT_OK(Insert('j', Cache::Priority::LOW)); + EXPECT_OK(this->Insert('i', Cache::Priority::LOW)); + EXPECT_OK(this->Insert('j', Cache::Priority::LOW)); - EXPECT_FALSE(Lookup('b', /*use*/ false)); - EXPECT_TRUE(Lookup('c', /*use*/ false)); - EXPECT_FALSE(Lookup('e', /*use*/ false)); - EXPECT_TRUE(Lookup('f', /*use*/ false)); + EXPECT_FALSE(this->Lookup('b', /*use*/ false)); + EXPECT_TRUE(this->Lookup('c', /*use*/ false)); + EXPECT_FALSE(this->Lookup('e', /*use*/ false)); + EXPECT_TRUE(this->Lookup('f', /*use*/ false)); // Mark g & h useful once again - EXPECT_TRUE(Lookup('g', /*use*/ true)); - EXPECT_TRUE(Lookup('h', /*use*/ true)); - EXPECT_TRUE(Lookup('i', /*use*/ false)); - EXPECT_TRUE(Lookup('j', /*use*/ false)); + EXPECT_TRUE(this->Lookup('g', /*use*/ true)); + EXPECT_TRUE(this->Lookup('h', /*use*/ true)); + EXPECT_TRUE(this->Lookup('i', /*use*/ false)); + EXPECT_TRUE(this->Lookup('j', /*use*/ false)); // Then old HIGH entries - EXPECT_OK(Insert('k', Cache::Priority::LOW)); - EXPECT_OK(Insert('l', Cache::Priority::LOW)); - - EXPECT_FALSE(Lookup('c', /*use*/ false)); - EXPECT_FALSE(Lookup('f', /*use*/ false)); - EXPECT_TRUE(Lookup('g', /*use*/ false)); - EXPECT_TRUE(Lookup('h', /*use*/ false)); - EXPECT_TRUE(Lookup('i', /*use*/ false)); - EXPECT_TRUE(Lookup('j', /*use*/ false)); - EXPECT_TRUE(Lookup('k', /*use*/ false)); - EXPECT_TRUE(Lookup('l', /*use*/ false)); + EXPECT_OK(this->Insert('k', Cache::Priority::LOW)); + EXPECT_OK(this->Insert('l', Cache::Priority::LOW)); + + EXPECT_FALSE(this->Lookup('c', /*use*/ false)); + EXPECT_FALSE(this->Lookup('f', /*use*/ false)); + EXPECT_TRUE(this->Lookup('g', /*use*/ false)); + EXPECT_TRUE(this->Lookup('h', /*use*/ false)); + EXPECT_TRUE(this->Lookup('i', /*use*/ false)); + EXPECT_TRUE(this->Lookup('j', /*use*/ false)); + EXPECT_TRUE(this->Lookup('k', /*use*/ false)); + EXPECT_TRUE(this->Lookup('l', /*use*/ false)); // Then the (roughly) least recently useful - EXPECT_OK(Insert('m', Cache::Priority::HIGH)); - EXPECT_OK(Insert('n', Cache::Priority::HIGH)); + EXPECT_OK(this->Insert('m', Cache::Priority::HIGH)); + EXPECT_OK(this->Insert('n', Cache::Priority::HIGH)); - EXPECT_TRUE(Lookup('g', /*use*/ false)); - EXPECT_TRUE(Lookup('h', /*use*/ false)); - EXPECT_FALSE(Lookup('i', /*use*/ false)); - EXPECT_FALSE(Lookup('j', /*use*/ false)); - EXPECT_TRUE(Lookup('k', /*use*/ false)); - EXPECT_TRUE(Lookup('l', /*use*/ false)); + EXPECT_TRUE(this->Lookup('g', /*use*/ false)); + EXPECT_TRUE(this->Lookup('h', /*use*/ false)); + EXPECT_FALSE(this->Lookup('i', /*use*/ false)); + EXPECT_FALSE(this->Lookup('j', /*use*/ false)); + EXPECT_TRUE(this->Lookup('k', /*use*/ false)); + EXPECT_TRUE(this->Lookup('l', /*use*/ false)); // Now try changing capacity down - shard_->SetCapacity(4); + shard.SetCapacity(4); // Insert to ensure evictions happen - EXPECT_OK(Insert('o', Cache::Priority::LOW)); - EXPECT_OK(Insert('p', Cache::Priority::LOW)); - - EXPECT_FALSE(Lookup('g', /*use*/ false)); - EXPECT_FALSE(Lookup('h', /*use*/ false)); - EXPECT_FALSE(Lookup('k', /*use*/ false)); - EXPECT_FALSE(Lookup('l', /*use*/ false)); - EXPECT_TRUE(Lookup('m', /*use*/ false)); - EXPECT_TRUE(Lookup('n', /*use*/ false)); - EXPECT_TRUE(Lookup('o', /*use*/ false)); - EXPECT_TRUE(Lookup('p', /*use*/ false)); + EXPECT_OK(this->Insert('o', Cache::Priority::LOW)); + EXPECT_OK(this->Insert('p', Cache::Priority::LOW)); + + EXPECT_FALSE(this->Lookup('g', /*use*/ false)); + EXPECT_FALSE(this->Lookup('h', /*use*/ false)); + EXPECT_FALSE(this->Lookup('k', /*use*/ false)); + EXPECT_FALSE(this->Lookup('l', /*use*/ false)); + EXPECT_TRUE(this->Lookup('m', /*use*/ false)); + EXPECT_TRUE(this->Lookup('n', /*use*/ false)); + EXPECT_TRUE(this->Lookup('o', /*use*/ false)); + EXPECT_TRUE(this->Lookup('p', /*use*/ false)); // Now try changing capacity up - EXPECT_TRUE(Lookup('m', /*use*/ true)); - EXPECT_TRUE(Lookup('n', /*use*/ true)); - shard_->SetCapacity(6); - EXPECT_OK(Insert('q', Cache::Priority::HIGH)); - EXPECT_OK(Insert('r', Cache::Priority::HIGH)); - EXPECT_OK(Insert('s', Cache::Priority::HIGH)); - EXPECT_OK(Insert('t', Cache::Priority::HIGH)); - - EXPECT_FALSE(Lookup('o', /*use*/ false)); - EXPECT_FALSE(Lookup('p', /*use*/ false)); - EXPECT_TRUE(Lookup('m', /*use*/ false)); - EXPECT_TRUE(Lookup('n', /*use*/ false)); - EXPECT_TRUE(Lookup('q', /*use*/ false)); - EXPECT_TRUE(Lookup('r', /*use*/ false)); - EXPECT_TRUE(Lookup('s', /*use*/ false)); - EXPECT_TRUE(Lookup('t', /*use*/ false)); + EXPECT_TRUE(this->Lookup('m', /*use*/ true)); + EXPECT_TRUE(this->Lookup('n', /*use*/ true)); + shard.SetCapacity(6); + EXPECT_OK(this->Insert('q', Cache::Priority::HIGH)); + EXPECT_OK(this->Insert('r', Cache::Priority::HIGH)); + EXPECT_OK(this->Insert('s', Cache::Priority::HIGH)); + EXPECT_OK(this->Insert('t', Cache::Priority::HIGH)); + + EXPECT_FALSE(this->Lookup('o', /*use*/ false)); + EXPECT_FALSE(this->Lookup('p', /*use*/ false)); + EXPECT_TRUE(this->Lookup('m', /*use*/ false)); + EXPECT_TRUE(this->Lookup('n', /*use*/ false)); + EXPECT_TRUE(this->Lookup('q', /*use*/ false)); + EXPECT_TRUE(this->Lookup('r', /*use*/ false)); + EXPECT_TRUE(this->Lookup('s', /*use*/ false)); + EXPECT_TRUE(this->Lookup('t', /*use*/ false)); + } +} + +TYPED_TEST(ClockCacheTest, ClockEvictionEffortCapTest) { + using HandleImpl = typename ClockCacheTest::Shard::HandleImpl; + for (bool strict_capacity_limit : {true, false}) { + SCOPED_TRACE("strict_capacity_limit = " + + std::to_string(strict_capacity_limit)); + for (int eec : {-42, 0, 1, 10, 100, 1000}) { + SCOPED_TRACE("eviction_effort_cap = " + std::to_string(eec)); + constexpr size_t kCapacity = 1000; + // Start with much larger capacity to ensure that we can go way over + // capacity without reaching table occupancy limit. + this->NewShard(3 * kCapacity, strict_capacity_limit, eec); + auto& shard = *this->shard_; + shard.SetCapacity(kCapacity); + + // Nearly fill the cache with pinned entries, then add a bunch of + // non-pinned entries. eviction_effort_cap should affect how many + // evictable entries are present beyond the cache capacity, despite + // being evictable. + constexpr size_t kCount = kCapacity - 1; + std::unique_ptr ha { new HandleImpl* [kCount] {} }; + for (size_t i = 0; i < 2 * kCount; ++i) { + UniqueId64x2 hkey = this->CheapHash(i); + ASSERT_OK(shard.Insert( + this->TestKey(hkey), hkey, nullptr /*value*/, &kNoopCacheItemHelper, + 1 /*charge*/, i < kCount ? &ha[i] : nullptr, Cache::Priority::LOW)); + } + + if (strict_capacity_limit) { + // If strict_capacity_limit is enabled, the cache will never exceed its + // capacity + EXPECT_EQ(shard.GetOccupancyCount(), kCapacity); + } else { + // Rough inverse relationship between cap and possible memory + // explosion, which shows up as increased table occupancy count. + int effective_eec = std::max(int{1}, eec) + 1; + EXPECT_NEAR(shard.GetOccupancyCount() * 1.0, + kCount * (1 + 1.4 / effective_eec), + kCount * (0.6 / effective_eec) + 1.0); + } + + for (size_t i = 0; i < kCount; ++i) { + shard.Release(ha[i]); + } + } } } @@ -682,66 +751,72 @@ const Cache::CacheItemHelper kDeleteCounterHelper{ } // namespace // Testing calls to CorrectNearOverflow in Release -TEST_F(ClockCacheTest, ClockCounterOverflowTest) { - NewShard(6, /*strict_capacity_limit*/ false); +TYPED_TEST(ClockCacheTest, ClockCounterOverflowTest) { + this->NewShard(6, /*strict_capacity_limit*/ false); + auto& shard = *this->shard_; + using HandleImpl = typename ClockCacheTest::Shard::HandleImpl; + HandleImpl* h; DeleteCounter val; - UniqueId64x2 hkey = TestHashedKey('x'); - ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &val, &kDeleteCounterHelper, 1, - &h, Cache::Priority::HIGH)); + UniqueId64x2 hkey = this->TestHashedKey('x'); + ASSERT_OK(shard.Insert(this->TestKey(hkey), hkey, &val, &kDeleteCounterHelper, + 1, &h, Cache::Priority::HIGH)); // Some large number outstanding - shard_->TEST_RefN(h, 123456789); + shard.TEST_RefN(h, 123456789); // Simulate many lookup/ref + release, plenty to overflow counters for (int i = 0; i < 10000; ++i) { - shard_->TEST_RefN(h, 1234567); - shard_->TEST_ReleaseN(h, 1234567); + shard.TEST_RefN(h, 1234567); + shard.TEST_ReleaseN(h, 1234567); } // Mark it invisible (to reach a different CorrectNearOverflow() in Release) - shard_->Erase(TestKey(hkey), hkey); + shard.Erase(this->TestKey(hkey), hkey); // Simulate many more lookup/ref + release (one-by-one would be too // expensive for unit test) for (int i = 0; i < 10000; ++i) { - shard_->TEST_RefN(h, 1234567); - shard_->TEST_ReleaseN(h, 1234567); + shard.TEST_RefN(h, 1234567); + shard.TEST_ReleaseN(h, 1234567); } // Free all but last 1 - shard_->TEST_ReleaseN(h, 123456789); + shard.TEST_ReleaseN(h, 123456789); // Still alive ASSERT_EQ(val.deleted, 0); // Free last ref, which will finalize erasure - shard_->Release(h); + shard.Release(h); // Deleted ASSERT_EQ(val.deleted, 1); } -TEST_F(ClockCacheTest, ClockTableFull) { +TYPED_TEST(ClockCacheTest, ClockTableFull) { // Force clock cache table to fill up (not usually allowed) in order // to test full probe sequence that is theoretically possible due to // parallel operations - NewShard(6, /*strict_capacity_limit*/ false); - size_t size = shard_->GetTableAddressCount(); + this->NewShard(6, /*strict_capacity_limit*/ false); + auto& shard = *this->shard_; + using HandleImpl = typename ClockCacheTest::Shard::HandleImpl; + + size_t size = shard.GetTableAddressCount(); ASSERT_LE(size + 3, 256); // for using char keys // Modify occupancy and capacity limits to attempt insert on full - shard_->TEST_MutableOccupancyLimit() = size + 100; - shard_->SetCapacity(size + 100); + shard.TEST_MutableOccupancyLimit() = size + 100; + shard.SetCapacity(size + 100); DeleteCounter val; std::vector handles; // NOTE: the three extra insertions should create standalone entries for (size_t i = 0; i < size + 3; ++i) { - UniqueId64x2 hkey = TestHashedKey(static_cast(i)); - ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &val, &kDeleteCounterHelper, - 1, &handles.emplace_back(), - Cache::Priority::HIGH)); + UniqueId64x2 hkey = this->TestHashedKey(static_cast(i)); + ASSERT_OK(shard.Insert(this->TestKey(hkey), hkey, &val, + &kDeleteCounterHelper, 1, &handles.emplace_back(), + Cache::Priority::HIGH)); } for (size_t i = 0; i < size + 3; ++i) { - UniqueId64x2 hkey = TestHashedKey(static_cast(i)); - HandleImpl* h = shard_->Lookup(TestKey(hkey), hkey); + UniqueId64x2 hkey = this->TestHashedKey(static_cast(i)); + HandleImpl* h = shard.Lookup(this->TestKey(hkey), hkey); if (i < size) { ASSERT_NE(h, nullptr); - shard_->Release(h); + shard.Release(h); } else { // Standalone entries not visible by lookup ASSERT_EQ(h, nullptr); @@ -750,7 +825,7 @@ TEST_F(ClockCacheTest, ClockTableFull) { for (size_t i = 0; i < size + 3; ++i) { ASSERT_NE(handles[i], nullptr); - shard_->Release(handles[i]); + shard.Release(handles[i]); if (i < size) { // Everything still in cache ASSERT_EQ(val.deleted, 0); @@ -761,8 +836,8 @@ TEST_F(ClockCacheTest, ClockTableFull) { } for (size_t i = size + 3; i > 0; --i) { - UniqueId64x2 hkey = TestHashedKey(static_cast(i - 1)); - shard_->Erase(TestKey(hkey), hkey); + UniqueId64x2 hkey = this->TestHashedKey(static_cast(i - 1)); + shard.Erase(this->TestKey(hkey), hkey); if (i - 1 > size) { ASSERT_EQ(val.deleted, 3); } else { @@ -773,78 +848,81 @@ TEST_F(ClockCacheTest, ClockTableFull) { // This test is mostly to exercise some corner case logic, by forcing two // keys to have the same hash, and more -TEST_F(ClockCacheTest, CollidingInsertEraseTest) { - NewShard(6, /*strict_capacity_limit*/ false); +TYPED_TEST(ClockCacheTest, CollidingInsertEraseTest) { + this->NewShard(6, /*strict_capacity_limit*/ false); + auto& shard = *this->shard_; + using HandleImpl = typename ClockCacheTest::Shard::HandleImpl; + DeleteCounter val; - UniqueId64x2 hkey1 = TestHashedKey('x'); - Slice key1 = TestKey(hkey1); - UniqueId64x2 hkey2 = TestHashedKey('y'); - Slice key2 = TestKey(hkey2); - UniqueId64x2 hkey3 = TestHashedKey('z'); - Slice key3 = TestKey(hkey3); + UniqueId64x2 hkey1 = this->TestHashedKey('x'); + Slice key1 = this->TestKey(hkey1); + UniqueId64x2 hkey2 = this->TestHashedKey('y'); + Slice key2 = this->TestKey(hkey2); + UniqueId64x2 hkey3 = this->TestHashedKey('z'); + Slice key3 = this->TestKey(hkey3); HandleImpl* h1; - ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, &h1, - Cache::Priority::HIGH)); + ASSERT_OK(shard.Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, &h1, + Cache::Priority::HIGH)); HandleImpl* h2; - ASSERT_OK(shard_->Insert(key2, hkey2, &val, &kDeleteCounterHelper, 1, &h2, - Cache::Priority::HIGH)); + ASSERT_OK(shard.Insert(key2, hkey2, &val, &kDeleteCounterHelper, 1, &h2, + Cache::Priority::HIGH)); HandleImpl* h3; - ASSERT_OK(shard_->Insert(key3, hkey3, &val, &kDeleteCounterHelper, 1, &h3, - Cache::Priority::HIGH)); + ASSERT_OK(shard.Insert(key3, hkey3, &val, &kDeleteCounterHelper, 1, &h3, + Cache::Priority::HIGH)); // Can repeatedly lookup+release despite the hash collision HandleImpl* tmp_h; for (bool erase_if_last_ref : {true, false}) { // but not last ref - tmp_h = shard_->Lookup(key1, hkey1); + tmp_h = shard.Lookup(key1, hkey1); ASSERT_EQ(h1, tmp_h); - ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref)); - tmp_h = shard_->Lookup(key2, hkey2); + tmp_h = shard.Lookup(key2, hkey2); ASSERT_EQ(h2, tmp_h); - ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref)); - tmp_h = shard_->Lookup(key3, hkey3); + tmp_h = shard.Lookup(key3, hkey3); ASSERT_EQ(h3, tmp_h); - ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref)); } // Make h1 invisible - shard_->Erase(key1, hkey1); + shard.Erase(key1, hkey1); // Redundant erase - shard_->Erase(key1, hkey1); + shard.Erase(key1, hkey1); // All still alive ASSERT_EQ(val.deleted, 0); // Invisible to Lookup - tmp_h = shard_->Lookup(key1, hkey1); + tmp_h = shard.Lookup(key1, hkey1); ASSERT_EQ(nullptr, tmp_h); // Can still find h2, h3 for (bool erase_if_last_ref : {true, false}) { // but not last ref - tmp_h = shard_->Lookup(key2, hkey2); + tmp_h = shard.Lookup(key2, hkey2); ASSERT_EQ(h2, tmp_h); - ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref)); - tmp_h = shard_->Lookup(key3, hkey3); + tmp_h = shard.Lookup(key3, hkey3); ASSERT_EQ(h3, tmp_h); - ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref)); } // Also Insert with invisible entry there - ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, nullptr, - Cache::Priority::HIGH)); - tmp_h = shard_->Lookup(key1, hkey1); + ASSERT_OK(shard.Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, nullptr, + Cache::Priority::HIGH)); + tmp_h = shard.Lookup(key1, hkey1); // Found but distinct handle ASSERT_NE(nullptr, tmp_h); ASSERT_NE(h1, tmp_h); - ASSERT_TRUE(shard_->Release(tmp_h, /*erase_if_last_ref*/ true)); + ASSERT_TRUE(shard.Release(tmp_h, /*erase_if_last_ref*/ true)); // tmp_h deleted ASSERT_EQ(val.deleted--, 1); // Release last ref on h1 (already invisible) - ASSERT_TRUE(shard_->Release(h1, /*erase_if_last_ref*/ false)); + ASSERT_TRUE(shard.Release(h1, /*erase_if_last_ref*/ false)); // h1 deleted ASSERT_EQ(val.deleted--, 1); @@ -852,57 +930,57 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) { // Can still find h2, h3 for (bool erase_if_last_ref : {true, false}) { // but not last ref - tmp_h = shard_->Lookup(key2, hkey2); + tmp_h = shard.Lookup(key2, hkey2); ASSERT_EQ(h2, tmp_h); - ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref)); - tmp_h = shard_->Lookup(key3, hkey3); + tmp_h = shard.Lookup(key3, hkey3); ASSERT_EQ(h3, tmp_h); - ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref)); } // Release last ref on h2 - ASSERT_FALSE(shard_->Release(h2, /*erase_if_last_ref*/ false)); + ASSERT_FALSE(shard.Release(h2, /*erase_if_last_ref*/ false)); // h2 still not deleted (unreferenced in cache) ASSERT_EQ(val.deleted, 0); // Can still find it - tmp_h = shard_->Lookup(key2, hkey2); + tmp_h = shard.Lookup(key2, hkey2); ASSERT_EQ(h2, tmp_h); // Release last ref on h2, with erase - ASSERT_TRUE(shard_->Release(h2, /*erase_if_last_ref*/ true)); + ASSERT_TRUE(shard.Release(h2, /*erase_if_last_ref*/ true)); // h2 deleted ASSERT_EQ(val.deleted--, 1); - tmp_h = shard_->Lookup(key2, hkey2); + tmp_h = shard.Lookup(key2, hkey2); ASSERT_EQ(nullptr, tmp_h); // Can still find h3 for (bool erase_if_last_ref : {true, false}) { // but not last ref - tmp_h = shard_->Lookup(key3, hkey3); + tmp_h = shard.Lookup(key3, hkey3); ASSERT_EQ(h3, tmp_h); - ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref)); } // Release last ref on h3, without erase - ASSERT_FALSE(shard_->Release(h3, /*erase_if_last_ref*/ false)); + ASSERT_FALSE(shard.Release(h3, /*erase_if_last_ref*/ false)); // h3 still not deleted (unreferenced in cache) ASSERT_EQ(val.deleted, 0); // Explicit erase - shard_->Erase(key3, hkey3); + shard.Erase(key3, hkey3); // h3 deleted ASSERT_EQ(val.deleted--, 1); - tmp_h = shard_->Lookup(key3, hkey3); + tmp_h = shard.Lookup(key3, hkey3); ASSERT_EQ(nullptr, tmp_h); } // This uses the public API to effectively test CalcHashBits etc. -TEST_F(ClockCacheTest, TableSizesTest) { +TYPED_TEST(ClockCacheTest, TableSizesTest) { for (size_t est_val_size : {1U, 5U, 123U, 2345U, 345678U}) { SCOPED_TRACE("est_val_size = " + std::to_string(est_val_size)); for (double est_count : {1.1, 2.2, 511.9, 512.1, 2345.0}) { @@ -915,8 +993,10 @@ TEST_F(ClockCacheTest, TableSizesTest) { /*memory_allocator*/ nullptr, kDontChargeCacheMetadata) .MakeSharedCache(); // Table sizes are currently only powers of two - EXPECT_GE(cache->GetTableAddressCount(), est_count / kLoadFactor); - EXPECT_LE(cache->GetTableAddressCount(), est_count / kLoadFactor * 2.0); + EXPECT_GE(cache->GetTableAddressCount(), + est_count / FixedHyperClockTable::kLoadFactor); + EXPECT_LE(cache->GetTableAddressCount(), + est_count / FixedHyperClockTable::kLoadFactor * 2.0); EXPECT_EQ(cache->GetUsage(), 0); // kFullChargeMetaData @@ -933,9 +1013,10 @@ TEST_F(ClockCacheTest, TableSizesTest) { double est_count_after_meta = (capacity - cache->GetUsage()) * 1.0 / est_val_size; EXPECT_GE(cache->GetTableAddressCount(), - est_count_after_meta / kLoadFactor); - EXPECT_LE(cache->GetTableAddressCount(), - est_count_after_meta / kLoadFactor * 2.0); + est_count_after_meta / FixedHyperClockTable::kLoadFactor); + EXPECT_LE( + cache->GetTableAddressCount(), + est_count_after_meta / FixedHyperClockTable::kLoadFactor * 2.0); } } } @@ -958,13 +1039,14 @@ class TestSecondaryCache : public SecondaryCache { using ResultMap = std::unordered_map; - explicit TestSecondaryCache(size_t capacity) + explicit TestSecondaryCache(size_t capacity, bool insert_saved = false) : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */, nullptr, kDefaultToAdaptiveMutex, kDontChargeCacheMetadata)), num_inserts_(0), num_lookups_(0), - inject_failure_(false) {} + inject_failure_(false), + insert_saved_(insert_saved) {} const char* Name() const override { return "TestSecondaryCache"; } @@ -973,7 +1055,8 @@ class TestSecondaryCache : public SecondaryCache { void ResetInjectFailure() { inject_failure_ = false; } Status Insert(const Slice& key, Cache::ObjectPtr value, - const Cache::CacheItemHelper* helper) override { + const Cache::CacheItemHelper* helper, + bool /*force_insert*/) override { if (inject_failure_) { return Status::Corruption("Insertion Data Corrupted"); } @@ -994,10 +1077,22 @@ class TestSecondaryCache : public SecondaryCache { return cache_.Insert(key, buf, size); } + Status InsertSaved(const Slice& key, const Slice& saved, + CompressionType /*type*/ = kNoCompression, + CacheTier /*source*/ = CacheTier::kVolatileTier) override { + if (insert_saved_) { + return Insert(key, const_cast(&saved), &kSliceCacheItemHelper, + /*force_insert=*/true); + } else { + return Status::OK(); + } + } + std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool /*wait*/, - bool /*advise_erase*/, bool& kept_in_sec_cache) override { + bool /*advise_erase*/, Statistics* /*stats*/, + bool& kept_in_sec_cache) override { std::string key_str = key.ToString(); TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str); @@ -1022,7 +1117,8 @@ class TestSecondaryCache : public SecondaryCache { char* ptr = cache_.Value(handle); size_t size = DecodeFixed64(ptr); ptr += sizeof(uint64_t); - s = helper->create_cb(Slice(ptr, size), create_context, + s = helper->create_cb(Slice(ptr, size), kNoCompression, + CacheTier::kVolatileTier, create_context, /*alloc*/ nullptr, &value, &charge); } if (s.ok()) { @@ -1111,6 +1207,7 @@ class TestSecondaryCache : public SecondaryCache { uint32_t num_inserts_; uint32_t num_lookups_; bool inject_failure_; + bool insert_saved_; std::string ckey_prefix_; ResultMap result_map_; }; @@ -1141,7 +1238,7 @@ INSTANTIATE_TEST_CASE_P(DBSecondaryCacheTest, DBSecondaryCacheTest, TEST_P(BasicSecondaryCacheTest, BasicTest) { std::shared_ptr secondary_cache = - std::make_shared(4096); + std::make_shared(4096, true); std::shared_ptr cache = NewCache(1024 /* capacity */, 0 /* num_shard_bits */, false /* strict_capacity_limit */, secondary_cache); @@ -1198,7 +1295,7 @@ TEST_P(BasicSecondaryCacheTest, BasicTest) { TEST_P(BasicSecondaryCacheTest, StatsTest) { std::shared_ptr secondary_cache = - std::make_shared(4096); + std::make_shared(4096, true); std::shared_ptr cache = NewCache(1024 /* capacity */, 0 /* num_shard_bits */, false /* strict_capacity_limit */, secondary_cache); @@ -1252,7 +1349,7 @@ TEST_P(BasicSecondaryCacheTest, StatsTest) { TEST_P(BasicSecondaryCacheTest, BasicFailTest) { std::shared_ptr secondary_cache = - std::make_shared(2048); + std::make_shared(2048, true); std::shared_ptr cache = NewCache(1024 /* capacity */, 0 /* num_shard_bits */, false /* strict_capacity_limit */, secondary_cache); @@ -1294,7 +1391,7 @@ TEST_P(BasicSecondaryCacheTest, BasicFailTest) { TEST_P(BasicSecondaryCacheTest, SaveFailTest) { std::shared_ptr secondary_cache = - std::make_shared(2048); + std::make_shared(2048, true); std::shared_ptr cache = NewCache(1024 /* capacity */, 0 /* num_shard_bits */, false /* strict_capacity_limit */, secondary_cache); @@ -1335,7 +1432,7 @@ TEST_P(BasicSecondaryCacheTest, SaveFailTest) { TEST_P(BasicSecondaryCacheTest, CreateFailTest) { std::shared_ptr secondary_cache = - std::make_shared(2048); + std::make_shared(2048, true); std::shared_ptr cache = NewCache(1024 /* capacity */, 0 /* num_shard_bits */, false /* strict_capacity_limit */, secondary_cache); @@ -1376,7 +1473,7 @@ TEST_P(BasicSecondaryCacheTest, CreateFailTest) { TEST_P(BasicSecondaryCacheTest, FullCapacityTest) { for (bool strict_capacity_limit : {false, true}) { std::shared_ptr secondary_cache = - std::make_shared(2048); + std::make_shared(2048, true); std::shared_ptr cache = NewCache(1024 /* capacity */, 0 /* num_shard_bits */, strict_capacity_limit, secondary_cache); @@ -1423,7 +1520,7 @@ TEST_P(BasicSecondaryCacheTest, FullCapacityTest) { k2.AsSlice(), GetHelper(CacheEntryRole::kDataBlock, /*secondary_compatible=*/false), /*context*/ this, Cache::Priority::LOW); - if (strict_capacity_limit || GetParam() == kHyperClock) { + if (strict_capacity_limit || IsHyperClock()) { ASSERT_NE(handle2, nullptr); cache->Release(handle2); ASSERT_EQ(secondary_cache->num_inserts(), 1u); @@ -1448,12 +1545,12 @@ TEST_P(BasicSecondaryCacheTest, FullCapacityTest) { // CORRECTION: this is not quite right. block_1 can be inserted into the block // cache because strict_capacity_limit=false, but it is removed from the cache // in Release() because of being over-capacity, without demoting to secondary -// cache. HyperClockCache doesn't check capacity on release (for efficiency) -// so can demote the over-capacity item to secondary cache. Also, we intend to -// add support for demotion in Release, but that currently causes too much -// unit test churn. +// cache. FixedHyperClockCache doesn't check capacity on release (for +// efficiency) so can demote the over-capacity item to secondary cache. Also, we +// intend to add support for demotion in Release, but that currently causes too +// much unit test churn. TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness1) { - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { // See CORRECTION above ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); return; @@ -1551,7 +1648,7 @@ TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness1) { // insert and cache block_1 in the block cache (this is the different place // from TestSecondaryCacheCorrectness1) TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness2) { - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); return; } @@ -1739,7 +1836,7 @@ TEST_P(DBSecondaryCacheTest, SecondaryCacheIntensiveTesting) { // if we try to insert block_1 to the block cache, it will always fails. Only // block_2 will be successfully inserted into the block cache. TEST_P(DBSecondaryCacheTest, SecondaryCacheFailureTest) { - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); return; } @@ -1849,7 +1946,7 @@ TEST_P(BasicSecondaryCacheTest, BasicWaitAllTest) { str.length())); } // Force all entries to be evicted to the secondary cache - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { // HCC doesn't respond immediately to SetCapacity for (int i = 9000; i < 9030; ++i) { ASSERT_OK(cache->Insert(ock.WithOffset(i).AsSlice(), nullptr, @@ -1904,7 +2001,7 @@ TEST_P(BasicSecondaryCacheTest, BasicWaitAllTest) { // a sync point callback in TestSecondaryCache::Lookup. We then control the // lookup result by setting the ResultMap. TEST_P(DBSecondaryCacheTest, TestSecondaryCacheMultiGet) { - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); return; } @@ -1995,8 +2092,9 @@ class CacheWithStats : public CacheWrapper { Status Insert(const Slice& key, Cache::ObjectPtr value, const CacheItemHelper* helper, size_t charge, - Handle** handle = nullptr, - Priority priority = Priority::LOW) override { + Handle** handle = nullptr, Priority priority = Priority::LOW, + const Slice& /*compressed*/ = Slice(), + CompressionType /*type*/ = kNoCompression) override { insert_count_++; return target_->Insert(key, value, helper, charge, handle, priority); } @@ -2089,7 +2187,7 @@ TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) { // we have a new cache it is empty, then, before we do the Get, we do the // dumpload std::shared_ptr secondary_cache = - std::make_shared(2048 * 1024); + std::make_shared(2048 * 1024, true); // This time with secondary cache base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */, false /* strict_capacity_limit */, secondary_cache); @@ -2245,7 +2343,7 @@ TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) { // we have a new cache it is empty, then, before we do the Get, we do the // dumpload std::shared_ptr secondary_cache = - std::make_shared(2048 * 1024); + std::make_shared(2048 * 1024, true); // This time with secondary_cache base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */, false /* strict_capacity_limit */, secondary_cache); @@ -2405,7 +2503,7 @@ TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionBasic) { // with new options, which set the lowest_used_cache_tier to // kNonVolatileBlockTier. So secondary cache will be used. TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionChange) { - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); return; } @@ -2500,7 +2598,7 @@ TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionChange) { // Two DB test. We create 2 DBs sharing the same block cache and secondary // cache. We diable the secondary cache option for DB2. TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionTwoDB) { - if (GetParam() == kHyperClock) { + if (IsHyperClock()) { ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); return; } diff --git a/cache/secondary_cache.cc b/cache/secondary_cache.cc index 8eba8b098..4439869f1 100644 --- a/cache/secondary_cache.cc +++ b/cache/secondary_cache.cc @@ -9,36 +9,4 @@ namespace ROCKSDB_NAMESPACE { -namespace { - -void NoopDelete(Cache::ObjectPtr, MemoryAllocator*) {} - -size_t SliceSize(Cache::ObjectPtr obj) { - return static_cast(obj)->size(); -} - -Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset, size_t length, - char* out) { - const Slice& slice = *static_cast(from_obj); - std::memcpy(out, slice.data() + from_offset, length); - return Status::OK(); -} - -Status FailCreate(const Slice&, Cache::CreateContext*, MemoryAllocator*, - Cache::ObjectPtr*, size_t*) { - return Status::NotSupported("Only for dumping data into SecondaryCache"); -} - -} // namespace - -Status SecondaryCache::InsertSaved(const Slice& key, const Slice& saved) { - static Cache::CacheItemHelper helper_no_secondary{CacheEntryRole::kMisc, - &NoopDelete}; - static Cache::CacheItemHelper helper{ - CacheEntryRole::kMisc, &NoopDelete, &SliceSize, - &SliceSaveTo, &FailCreate, &helper_no_secondary}; - // NOTE: depends on Insert() being synchronous, not keeping pointer `&saved` - return Insert(key, const_cast(&saved), &helper); -} - } // namespace ROCKSDB_NAMESPACE diff --git a/cache/secondary_cache_adapter.cc b/cache/secondary_cache_adapter.cc index 06441a17a..6261b8ce6 100644 --- a/cache/secondary_cache_adapter.cc +++ b/cache/secondary_cache_adapter.cc @@ -5,7 +5,11 @@ #include "cache/secondary_cache_adapter.h" +#include + +#include "cache/tiered_secondary_cache.h" #include "monitoring/perf_context_imp.h" +#include "test_util/sync_point.h" #include "util/cast_util.h" namespace ROCKSDB_NAMESPACE { @@ -17,6 +21,7 @@ struct Dummy { }; const Dummy kDummy{}; Cache::ObjectPtr const kDummyObj = const_cast(&kDummy); +const char* kTieredCacheName = "TieredCache"; } // namespace // When CacheWithSecondaryAdapter is constructed with the distribute_cache_res @@ -73,13 +78,19 @@ Cache::ObjectPtr const kDummyObj = const_cast(&kDummy); // CacheWithSecondaryAdapter::CacheWithSecondaryAdapter( std::shared_ptr target, - std::shared_ptr secondary_cache, bool distribute_cache_res) + std::shared_ptr secondary_cache, + TieredAdmissionPolicy adm_policy, bool distribute_cache_res) : CacheWrapper(std::move(target)), secondary_cache_(std::move(secondary_cache)), - distribute_cache_res_(distribute_cache_res) { - target_->SetEvictionCallback([this](const Slice& key, Handle* handle) { - return EvictionHandler(key, handle); - }); + adm_policy_(adm_policy), + distribute_cache_res_(distribute_cache_res), + placeholder_usage_(0), + reserved_usage_(0), + sec_reserved_(0) { + target_->SetEvictionCallback( + [this](const Slice& key, Handle* handle, bool was_hit) { + return EvictionHandler(key, handle, was_hit); + }); if (distribute_cache_res_) { size_t sec_capacity = 0; pri_cache_res_ = std::make_shared( @@ -108,20 +119,27 @@ CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() { size_t sec_capacity = 0; Status s = secondary_cache_->GetCapacity(sec_capacity); assert(s.ok()); - assert(pri_cache_res_->GetTotalReservedCacheSize() == sec_capacity); + assert(placeholder_usage_ == 0); + assert(reserved_usage_ == 0); + assert(pri_cache_res_->GetTotalMemoryUsed() == sec_capacity); } #endif // NDEBUG } bool CacheWithSecondaryAdapter::EvictionHandler(const Slice& key, - Handle* handle) { + Handle* handle, bool was_hit) { auto helper = GetCacheItemHelper(handle); - if (helper->IsSecondaryCacheCompatible()) { + if (helper->IsSecondaryCacheCompatible() && + adm_policy_ != TieredAdmissionPolicy::kAdmPolicyThreeQueue) { auto obj = target_->Value(handle); // Ignore dummy entry if (obj != kDummyObj) { + bool hit = false; + if (adm_policy_ == TieredAdmissionPolicy::kAdmPolicyAllowCacheHits) { + hit = was_hit; + } // Spill into secondary cache. - secondary_cache_->Insert(key, obj, helper).PermitUncheckedError(); + secondary_cache_->Insert(key, obj, helper, hit).PermitUncheckedError(); } } // Never takes ownership of obj @@ -218,14 +236,42 @@ Cache::Handle* CacheWithSecondaryAdapter::Promote( Status CacheWithSecondaryAdapter::Insert(const Slice& key, ObjectPtr value, const CacheItemHelper* helper, size_t charge, Handle** handle, - Priority priority) { + Priority priority, + const Slice& compressed_value, + CompressionType type) { Status s = target_->Insert(key, value, helper, charge, handle, priority); - if (s.ok() && value == nullptr && distribute_cache_res_) { - size_t sec_charge = static_cast(charge * (sec_cache_res_ratio_)); - s = secondary_cache_->Deflate(sec_charge); - assert(s.ok()); - s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/false); - assert(s.ok()); + if (s.ok() && value == nullptr && distribute_cache_res_ && handle) { + charge = target_->GetCharge(*handle); + + MutexLock l(&cache_res_mutex_); + placeholder_usage_ += charge; + // Check if total placeholder reservation is more than the overall + // cache capacity. If it is, then we don't try to charge the + // secondary cache because we don't want to overcharge it (beyond + // its capacity). + // In order to make this a bit more lightweight, we also check if + // the difference between placeholder_usage_ and reserved_usage_ is + // atleast kReservationChunkSize and avoid any adjustments if not. + if ((placeholder_usage_ <= target_->GetCapacity()) && + ((placeholder_usage_ - reserved_usage_) >= kReservationChunkSize)) { + reserved_usage_ = placeholder_usage_ & ~(kReservationChunkSize - 1); + size_t new_sec_reserved = + static_cast(reserved_usage_ * sec_cache_res_ratio_); + size_t sec_charge = new_sec_reserved - sec_reserved_; + s = secondary_cache_->Deflate(sec_charge); + assert(s.ok()); + s = pri_cache_res_->UpdateCacheReservation(sec_charge, + /*increase=*/false); + assert(s.ok()); + sec_reserved_ += sec_charge; + } + } + // Warm up the secondary cache with the compressed block. The secondary + // cache may choose to ignore it based on the admission policy. + if (value != nullptr && !compressed_value.empty() && + adm_policy_ == TieredAdmissionPolicy::kAdmPolicyThreeQueue) { + Status status = secondary_cache_->InsertSaved(key, compressed_value, type); + assert(status.ok() || status.IsNotSupported()); } return s; @@ -248,7 +294,8 @@ Cache::Handle* CacheWithSecondaryAdapter::Lookup(const Slice& key, bool kept_in_sec_cache = false; std::unique_ptr secondary_handle = secondary_cache_->Lookup(key, helper, create_context, /*wait*/ true, - found_dummy_entry, /*out*/ kept_in_sec_cache); + found_dummy_entry, stats, + /*out*/ kept_in_sec_cache); if (secondary_handle) { result = Promote(std::move(secondary_handle), key, helper, priority, stats, found_dummy_entry, kept_in_sec_cache); @@ -263,11 +310,27 @@ bool CacheWithSecondaryAdapter::Release(Handle* handle, ObjectPtr v = target_->Value(handle); if (v == nullptr && distribute_cache_res_) { size_t charge = target_->GetCharge(handle); - size_t sec_charge = static_cast(charge * (sec_cache_res_ratio_)); - Status s = secondary_cache_->Inflate(sec_charge); - assert(s.ok()); - s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/true); - assert(s.ok()); + + MutexLock l(&cache_res_mutex_); + placeholder_usage_ -= charge; + // Check if total placeholder reservation is more than the overall + // cache capacity. If it is, then we do nothing as reserved_usage_ must + // be already maxed out + if ((placeholder_usage_ <= target_->GetCapacity()) && + (placeholder_usage_ < reserved_usage_)) { + // Adjust reserved_usage_ in chunks of kReservationChunkSize, so + // we don't hit this slow path too often. + reserved_usage_ = placeholder_usage_ & ~(kReservationChunkSize - 1); + size_t new_sec_reserved = + static_cast(reserved_usage_ * sec_cache_res_ratio_); + size_t sec_charge = sec_reserved_ - new_sec_reserved; + Status s = secondary_cache_->Inflate(sec_charge); + assert(s.ok()); + s = pri_cache_res_->UpdateCacheReservation(sec_charge, + /*increase=*/true); + assert(s.ok()); + sec_reserved_ -= sec_charge; + } } } return target_->Release(handle, erase_if_last_ref); @@ -286,10 +349,10 @@ void CacheWithSecondaryAdapter::StartAsyncLookupOnMySecondary( assert(async_handle.result_handle == nullptr); std::unique_ptr secondary_handle = - secondary_cache_->Lookup(async_handle.key, async_handle.helper, - async_handle.create_context, /*wait*/ false, - async_handle.found_dummy_entry, - /*out*/ async_handle.kept_in_sec_cache); + secondary_cache_->Lookup( + async_handle.key, async_handle.helper, async_handle.create_context, + /*wait*/ false, async_handle.found_dummy_entry, async_handle.stats, + /*out*/ async_handle.kept_in_sec_cache); if (secondary_handle) { // TODO with stacked secondaries: Check & process if already ready? async_handle.pending_handle = secondary_handle.release(); @@ -399,35 +462,279 @@ std::string CacheWithSecondaryAdapter::GetPrintableOptions() const { } const char* CacheWithSecondaryAdapter::Name() const { - // To the user, at least for now, configure the underlying cache with - // a secondary cache. So we pretend to be that cache - return target_->Name(); + if (distribute_cache_res_) { + return kTieredCacheName; + } else { + // To the user, at least for now, configure the underlying cache with + // a secondary cache. So we pretend to be that cache + return target_->Name(); + } +} + +// Update the total cache capacity. If we're distributing cache reservations +// to both primary and secondary, then update the pri_cache_res_reservation +// as well. At the moment, we don't have a good way of handling the case +// where the new capacity < total cache reservations. +void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) { + size_t sec_capacity = static_cast( + capacity * (distribute_cache_res_ ? sec_cache_res_ratio_ : 0.0)); + size_t old_sec_capacity = 0; + + if (distribute_cache_res_) { + MutexLock m(&cache_res_mutex_); + + Status s = secondary_cache_->GetCapacity(old_sec_capacity); + if (!s.ok()) { + return; + } + if (old_sec_capacity > sec_capacity) { + // We're shrinking the cache. We do things in the following order to + // avoid a temporary spike in usage over the configured capacity - + // 1. Lower the secondary cache capacity + // 2. Credit an equal amount (by decreasing pri_cache_res_) to the + // primary cache + // 3. Decrease the primary cache capacity to the total budget + s = secondary_cache_->SetCapacity(sec_capacity); + if (s.ok()) { + if (placeholder_usage_ > capacity) { + // Adjust reserved_usage_ down + reserved_usage_ = capacity & ~(kReservationChunkSize - 1); + } + size_t new_sec_reserved = + static_cast(reserved_usage_ * sec_cache_res_ratio_); + s = pri_cache_res_->UpdateCacheReservation( + (old_sec_capacity - sec_capacity) - + (sec_reserved_ - new_sec_reserved), + /*increase=*/false); + sec_reserved_ = new_sec_reserved; + assert(s.ok()); + target_->SetCapacity(capacity); + } + } else { + // We're expanding the cache. Do it in the following order to avoid + // unnecessary evictions - + // 1. Increase the primary cache capacity to total budget + // 2. Reserve additional memory in primary on behalf of secondary (by + // increasing pri_cache_res_ reservation) + // 3. Increase secondary cache capacity + target_->SetCapacity(capacity); + s = pri_cache_res_->UpdateCacheReservation( + sec_capacity - old_sec_capacity, + /*increase=*/true); + assert(s.ok()); + s = secondary_cache_->SetCapacity(sec_capacity); + assert(s.ok()); + } + } else { + // No cache reservation distribution. Just set the primary cache capacity. + target_->SetCapacity(capacity); + } +} + +Status CacheWithSecondaryAdapter::GetSecondaryCacheCapacity( + size_t& size) const { + return secondary_cache_->GetCapacity(size); +} + +Status CacheWithSecondaryAdapter::GetSecondaryCachePinnedUsage( + size_t& size) const { + Status s; + if (distribute_cache_res_) { + MutexLock m(&cache_res_mutex_); + size_t capacity = 0; + s = secondary_cache_->GetCapacity(capacity); + if (s.ok()) { + size = capacity - pri_cache_res_->GetTotalMemoryUsed(); + } else { + size = 0; + } + } else { + size = 0; + } + return s; +} + +// Update the secondary/primary allocation ratio (remember, the primary +// capacity is the total memory budget when distribute_cache_res_ is true). +// When the ratio changes, we may accumulate some error in the calculations +// for secondary cache inflate/deflate and pri_cache_res_ reservations. +// This is due to the rounding of the reservation amount. +// +// We rely on the current pri_cache_res_ total memory used to estimate the +// new secondary cache reservation after the ratio change. For this reason, +// once the ratio is lowered to 0.0 (effectively disabling the secondary +// cache and pri_cache_res_ total mem used going down to 0), we cannot +// increase the ratio and re-enable it, We might remove this limitation +// in the future. +Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio( + double compressed_secondary_ratio) { + if (!distribute_cache_res_) { + return Status::NotSupported(); + } + + MutexLock m(&cache_res_mutex_); + size_t pri_capacity = target_->GetCapacity(); + size_t sec_capacity = + static_cast(pri_capacity * compressed_secondary_ratio); + size_t old_sec_capacity; + Status s = secondary_cache_->GetCapacity(old_sec_capacity); + if (!s.ok()) { + return s; + } + + // Calculate the new secondary cache reservation + // reserved_usage_ will never be > the cache capacity, so we don't + // have to worry about adjusting it here. + sec_cache_res_ratio_ = compressed_secondary_ratio; + size_t new_sec_reserved = + static_cast(reserved_usage_ * sec_cache_res_ratio_); + if (sec_capacity > old_sec_capacity) { + // We're increasing the ratio, thus ending up with a larger secondary + // cache and a smaller usable primary cache capacity. Similar to + // SetCapacity(), we try to avoid a temporary increase in total usage + // beyond the configured capacity - + // 1. A higher secondary cache ratio means it gets a higher share of + // cache reservations. So first account for that by deflating the + // secondary cache + // 2. Increase pri_cache_res_ reservation to reflect the new secondary + // cache utilization (increase in capacity - increase in share of cache + // reservation) + // 3. Increase secondary cache capacity + s = secondary_cache_->Deflate(new_sec_reserved - sec_reserved_); + assert(s.ok()); + s = pri_cache_res_->UpdateCacheReservation( + (sec_capacity - old_sec_capacity) - (new_sec_reserved - sec_reserved_), + /*increase=*/true); + assert(s.ok()); + sec_reserved_ = new_sec_reserved; + s = secondary_cache_->SetCapacity(sec_capacity); + assert(s.ok()); + } else { + // We're shrinking the ratio. Try to avoid unnecessary evictions - + // 1. Lower the secondary cache capacity + // 2. Decrease pri_cache_res_ reservation to relect lower secondary + // cache utilization (decrease in capacity - decrease in share of cache + // reservations) + // 3. Inflate the secondary cache to give it back the reduction in its + // share of cache reservations + s = secondary_cache_->SetCapacity(sec_capacity); + if (s.ok()) { + s = pri_cache_res_->UpdateCacheReservation( + (old_sec_capacity - sec_capacity) - + (sec_reserved_ - new_sec_reserved), + /*increase=*/false); + assert(s.ok()); + s = secondary_cache_->Inflate(sec_reserved_ - new_sec_reserved); + assert(s.ok()); + sec_reserved_ = new_sec_reserved; + } + } + + return s; } -std::shared_ptr NewTieredVolatileCache( - TieredVolatileCacheOptions& opts) { - if (!opts.cache_opts) { +Status CacheWithSecondaryAdapter::UpdateAdmissionPolicy( + TieredAdmissionPolicy adm_policy) { + adm_policy_ = adm_policy; + return Status::OK(); +} + +std::shared_ptr NewTieredCache(const TieredCacheOptions& _opts) { + if (!_opts.cache_opts) { return nullptr; } + TieredCacheOptions opts = _opts; + { + bool valid_adm_policy = true; + + switch (_opts.adm_policy) { + case TieredAdmissionPolicy::kAdmPolicyAuto: + // Select an appropriate default policy + if (opts.adm_policy == TieredAdmissionPolicy::kAdmPolicyAuto) { + if (opts.nvm_sec_cache) { + opts.adm_policy = TieredAdmissionPolicy::kAdmPolicyThreeQueue; + } else { + opts.adm_policy = TieredAdmissionPolicy::kAdmPolicyPlaceholder; + } + } + break; + case TieredAdmissionPolicy::kAdmPolicyPlaceholder: + case TieredAdmissionPolicy::kAdmPolicyAllowCacheHits: + if (opts.nvm_sec_cache) { + valid_adm_policy = false; + } + break; + case TieredAdmissionPolicy::kAdmPolicyThreeQueue: + if (!opts.nvm_sec_cache) { + valid_adm_policy = false; + } + break; + default: + valid_adm_policy = false; + } + if (!valid_adm_policy) { + return nullptr; + } + } + std::shared_ptr cache; if (opts.cache_type == PrimaryCacheType::kCacheTypeLRU) { LRUCacheOptions cache_opts = *(static_cast_with_check( opts.cache_opts)); - cache_opts.capacity += opts.comp_cache_opts.capacity; + cache_opts.capacity = opts.total_capacity; + cache_opts.secondary_cache = nullptr; cache = cache_opts.MakeSharedCache(); } else if (opts.cache_type == PrimaryCacheType::kCacheTypeHCC) { HyperClockCacheOptions cache_opts = *(static_cast_with_check( opts.cache_opts)); + cache_opts.capacity = opts.total_capacity; + cache_opts.secondary_cache = nullptr; cache = cache_opts.MakeSharedCache(); } else { return nullptr; } std::shared_ptr sec_cache; + opts.comp_cache_opts.capacity = static_cast( + opts.total_capacity * opts.compressed_secondary_ratio); sec_cache = NewCompressedSecondaryCache(opts.comp_cache_opts); - return std::make_shared(cache, sec_cache, true); + if (opts.nvm_sec_cache) { + if (opts.adm_policy == TieredAdmissionPolicy::kAdmPolicyThreeQueue) { + sec_cache = std::make_shared( + sec_cache, opts.nvm_sec_cache, + TieredAdmissionPolicy::kAdmPolicyThreeQueue); + } else { + return nullptr; + } + } + + return std::make_shared( + cache, sec_cache, opts.adm_policy, /*distribute_cache_res=*/true); +} + +Status UpdateTieredCache(const std::shared_ptr& cache, + int64_t total_capacity, + double compressed_secondary_ratio, + TieredAdmissionPolicy adm_policy) { + if (!cache || strcmp(cache->Name(), kTieredCacheName)) { + return Status::InvalidArgument(); + } + CacheWithSecondaryAdapter* tiered_cache = + static_cast(cache.get()); + + Status s; + if (total_capacity > 0) { + tiered_cache->SetCapacity(total_capacity); + } + if (compressed_secondary_ratio >= 0.0 && compressed_secondary_ratio <= 1.0) { + s = tiered_cache->UpdateCacheReservationRatio(compressed_secondary_ratio); + } + if (adm_policy < TieredAdmissionPolicy::kAdmPolicyMax) { + s = tiered_cache->UpdateAdmissionPolicy(adm_policy); + } + return s; } } // namespace ROCKSDB_NAMESPACE diff --git a/cache/secondary_cache_adapter.h b/cache/secondary_cache_adapter.h index 4ac93898e..f0a514e78 100644 --- a/cache/secondary_cache_adapter.h +++ b/cache/secondary_cache_adapter.h @@ -15,14 +15,17 @@ class CacheWithSecondaryAdapter : public CacheWrapper { explicit CacheWithSecondaryAdapter( std::shared_ptr target, std::shared_ptr secondary_cache, + TieredAdmissionPolicy adm_policy = TieredAdmissionPolicy::kAdmPolicyAuto, bool distribute_cache_res = false); ~CacheWithSecondaryAdapter() override; - Status Insert(const Slice& key, ObjectPtr value, - const CacheItemHelper* helper, size_t charge, - Handle** handle = nullptr, - Priority priority = Priority::LOW) override; + Status Insert( + const Slice& key, ObjectPtr value, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW, + const Slice& compressed_value = Slice(), + CompressionType type = CompressionType::kNoCompression) override; Handle* Lookup(const Slice& key, const CacheItemHelper* helper, CreateContext* create_context, @@ -42,12 +45,24 @@ class CacheWithSecondaryAdapter : public CacheWrapper { const char* Name() const override; + void SetCapacity(size_t capacity) override; + + Status GetSecondaryCacheCapacity(size_t& size) const override; + + Status GetSecondaryCachePinnedUsage(size_t& size) const override; + + Status UpdateCacheReservationRatio(double ratio); + + Status UpdateAdmissionPolicy(TieredAdmissionPolicy adm_policy); + Cache* TEST_GetCache() { return target_.get(); } SecondaryCache* TEST_GetSecondaryCache() { return secondary_cache_.get(); } private: - bool EvictionHandler(const Slice& key, Handle* handle); + static constexpr size_t kReservationChunkSize = 1 << 20; + + bool EvictionHandler(const Slice& key, Handle* handle, bool was_hit); void StartAsyncLookupOnMySecondary(AsyncLookupHandle& async_handle); @@ -61,6 +76,7 @@ class CacheWithSecondaryAdapter : public CacheWrapper { void CleanupCacheObject(ObjectPtr obj, const CacheItemHelper* helper); std::shared_ptr secondary_cache_; + TieredAdmissionPolicy adm_policy_; // Whether to proportionally distribute cache memory reservations, i.e // placeholder entries with null value and a non-zero charge, across // the primary and secondary caches. @@ -71,6 +87,17 @@ class CacheWithSecondaryAdapter : public CacheWrapper { // Fraction of a cache memory reservation to be assigned to the secondary // cache double sec_cache_res_ratio_; + // Mutex for use when managing cache memory reservations. Should not be used + // for other purposes, as it may risk causing deadlocks. + mutable port::Mutex cache_res_mutex_; + // Total memory reserved by placeholder entriesin the cache + size_t placeholder_usage_; + // Total placeholoder memory charged to both the primary and secondary + // caches. Will be <= placeholder_usage_. + size_t reserved_usage_; + // Amount of memory reserved in the secondary cache. This should be + // reserved_usage_ * sec_cache_res_ratio_ in steady state. + size_t sec_reserved_; }; } // namespace ROCKSDB_NAMESPACE diff --git a/cache/sharded_cache.cc b/cache/sharded_cache.cc index cb8555b35..b270df751 100644 --- a/cache/sharded_cache.cc +++ b/cache/sharded_cache.cc @@ -38,7 +38,7 @@ uint32_t DetermineSeed(int32_t hash_seed_option) { return GetSliceHash(hostname) & kSeedMask; } else { // Fall back on something stable within the process. - return static_cast(gen.GetBaseUpper()) & kSeedMask; + return BitwiseAnd(gen.GetBaseUpper(), kSeedMask); } } else { // for kQuasiRandomHashSeed and fallback @@ -83,6 +83,16 @@ size_t ShardedCacheBase::GetCapacity() const { return capacity_; } +Status ShardedCacheBase::GetSecondaryCacheCapacity(size_t& size) const { + size = 0; + return Status::OK(); +} + +Status ShardedCacheBase::GetSecondaryCachePinnedUsage(size_t& size) const { + size = 0; + return Status::OK(); +} + bool ShardedCacheBase::HasStrictCapacityLimit() const { MutexLock l(&config_mutex_); return strict_capacity_limit_; diff --git a/cache/sharded_cache.h b/cache/sharded_cache.h index d78cfc246..b7ef723a1 100644 --- a/cache/sharded_cache.h +++ b/cache/sharded_cache.h @@ -99,6 +99,8 @@ class ShardedCacheBase : public Cache { bool HasStrictCapacityLimit() const override; size_t GetCapacity() const override; + Status GetSecondaryCacheCapacity(size_t& size) const override; + Status GetSecondaryCachePinnedUsage(size_t& size) const override; using Cache::GetUsage; size_t GetUsage(Handle* handle) const override; @@ -170,9 +172,12 @@ class ShardedCache : public ShardedCacheBase { [s_c_l](CacheShard* cs) { cs->SetStrictCapacityLimit(s_c_l); }); } - Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, - size_t charge, Handle** handle = nullptr, - Priority priority = Priority::LOW) override { + Status Insert( + const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW, + const Slice& /*compressed_value*/ = Slice(), + CompressionType /*type*/ = CompressionType::kNoCompression) override { assert(helper); HashVal hash = CacheShard::ComputeHash(key, hash_seed_); auto h_out = reinterpret_cast(handle); @@ -273,6 +278,14 @@ class ShardedCache : public ShardedCacheBase { } } + inline void ForEachShard( + const std::function& fn) const { + uint32_t num_shards = GetNumShards(); + for (uint32_t i = 0; i < num_shards; i++) { + fn(shards_ + i); + } + } + inline size_t SumOverShards( const std::function& fn) const { uint32_t num_shards = GetNumShards(); diff --git a/cache/tiered_secondary_cache.cc b/cache/tiered_secondary_cache.cc new file mode 100644 index 000000000..f7d5dd91d --- /dev/null +++ b/cache/tiered_secondary_cache.cc @@ -0,0 +1,125 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/tiered_secondary_cache.h" + +#include "monitoring/statistics_impl.h" + +namespace ROCKSDB_NAMESPACE { + +// Creation callback for use in the lookup path. It calls the upper layer +// create_cb to create the object, and optionally calls the compressed +// secondary cache InsertSaved to save the compressed block. If +// advise_erase is set, it means the primary cache wants the block to be +// erased in the secondary cache, so we skip calling InsertSaved. +// +// For the time being, we assume that all blocks in the nvm tier belong to +// the primary block cache (i.e CacheTier::kVolatileTier). That can be changed +// if we implement demotion from the compressed secondary cache to the nvm +// cache in the future. +Status TieredSecondaryCache::MaybeInsertAndCreate( + const Slice& data, CompressionType type, CacheTier source, + Cache::CreateContext* ctx, MemoryAllocator* allocator, + Cache::ObjectPtr* out_obj, size_t* out_charge) { + TieredSecondaryCache::CreateContext* context = + static_cast(ctx); + assert(source == CacheTier::kVolatileTier); + if (!context->advise_erase && type != kNoCompression) { + // Attempt to insert into compressed secondary cache + // TODO: Don't hardcode the source + context->comp_sec_cache->InsertSaved(*context->key, data, type, source) + .PermitUncheckedError(); + RecordTick(context->stats, COMPRESSED_SECONDARY_CACHE_PROMOTIONS); + } else { + RecordTick(context->stats, COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS); + } + // Primary cache will accept the object, so call its helper to create + // the object + return context->helper->create_cb(data, type, source, context->inner_ctx, + allocator, out_obj, out_charge); +} + +// The lookup first looks up in the compressed secondary cache. If its a miss, +// then the nvm cache lookup is called. The cache item helper and create +// context are wrapped in order to intercept the creation callback to make +// the decision on promoting to the compressed secondary cache. +std::unique_ptr TieredSecondaryCache::Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool wait, bool advise_erase, + Statistics* stats, bool& kept_in_sec_cache) { + bool dummy = false; + std::unique_ptr result = + target()->Lookup(key, helper, create_context, wait, advise_erase, stats, + /*kept_in_sec_cache=*/dummy); + // We never want the item to spill back into the secondary cache + kept_in_sec_cache = true; + if (result) { + assert(result->IsReady()); + return result; + } + + // If wait is true, then we can be a bit more efficient and avoid a memory + // allocation for the CReateContext. + const Cache::CacheItemHelper* outer_helper = + TieredSecondaryCache::GetHelper(); + if (wait) { + TieredSecondaryCache::CreateContext ctx; + ctx.key = &key; + ctx.advise_erase = advise_erase; + ctx.helper = helper; + ctx.inner_ctx = create_context; + ctx.comp_sec_cache = target(); + ctx.stats = stats; + + return nvm_sec_cache_->Lookup(key, outer_helper, &ctx, wait, advise_erase, + stats, kept_in_sec_cache); + } + + // If wait is false, i.e its an async lookup, we have to allocate a result + // handle for tracking purposes. Embed the CreateContext inside the handle + // so we need only allocate memory once instead of twice. + std::unique_ptr handle(new ResultHandle()); + handle->ctx()->key = &key; + handle->ctx()->advise_erase = advise_erase; + handle->ctx()->helper = helper; + handle->ctx()->inner_ctx = create_context; + handle->ctx()->comp_sec_cache = target(); + handle->ctx()->stats = stats; + handle->SetInnerHandle( + nvm_sec_cache_->Lookup(key, outer_helper, handle->ctx(), wait, + advise_erase, stats, kept_in_sec_cache)); + if (!handle->inner_handle()) { + handle.reset(); + } else { + result.reset(handle.release()); + } + + return result; +} + +// Call the nvm cache WaitAll to complete the lookups +void TieredSecondaryCache::WaitAll( + std::vector handles) { + std::vector nvm_handles; + std::vector my_handles; + nvm_handles.reserve(handles.size()); + for (auto handle : handles) { + // The handle could belong to the compressed secondary cache. Skip if + // that's the case. + if (handle->IsReady()) { + continue; + } + ResultHandle* hdl = static_cast(handle); + nvm_handles.push_back(hdl->inner_handle()); + my_handles.push_back(hdl); + } + nvm_sec_cache_->WaitAll(nvm_handles); + for (auto handle : my_handles) { + assert(handle->inner_handle()->IsReady()); + handle->Complete(); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/cache/tiered_secondary_cache.h b/cache/tiered_secondary_cache.h new file mode 100644 index 000000000..80542ba49 --- /dev/null +++ b/cache/tiered_secondary_cache.h @@ -0,0 +1,160 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/cache.h" +#include "rocksdb/secondary_cache.h" + +namespace ROCKSDB_NAMESPACE { + +// A SecondaryCache that implements stacking of a compressed secondary cache +// and a non-volatile (local flash) cache. It implements an admission +// policy of warming the bottommost tier (local flash) with compressed +// blocks from the SST on misses, and on hits in the bottommost tier, +// promoting to the compressed and/or primary block cache. The admission +// policies of the primary block cache and compressed secondary cache remain +// unchanged - promote on second access. There is no demotion ofablocks +// evicted from a tier. They are just discarded. +// +// In order to properly handle compressed blocks directly read from SSTs, and +// to allow writeback of blocks compressed by the compressed secondary +// cache in the future, we make use of the compression type and source +// cache tier arguments in InsertSaved. +class TieredSecondaryCache : public SecondaryCacheWrapper { + public: + TieredSecondaryCache(std::shared_ptr comp_sec_cache, + std::shared_ptr nvm_sec_cache, + TieredAdmissionPolicy adm_policy) + : SecondaryCacheWrapper(comp_sec_cache), nvm_sec_cache_(nvm_sec_cache) { +#ifndef NDEBUG + assert(adm_policy == TieredAdmissionPolicy::kAdmPolicyThreeQueue); +#else + (void)adm_policy; +#endif + } + + ~TieredSecondaryCache() override {} + + const char* Name() const override { return "TieredSecondaryCache"; } + + // This is a no-op as we currently don't allow demotion (i.e + // insertion by the upper layer) of evicted blocks. + virtual Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*obj*/, + const Cache::CacheItemHelper* /*helper*/, + bool /*force_insert*/) override { + return Status::OK(); + } + + // Warm up the nvm tier directly + virtual Status InsertSaved( + const Slice& key, const Slice& saved, + CompressionType type = CompressionType::kNoCompression, + CacheTier source = CacheTier::kVolatileTier) override { + return nvm_sec_cache_->InsertSaved(key, saved, type, source); + } + + virtual std::unique_ptr Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool wait, bool advise_erase, + Statistics* stats, bool& kept_in_sec_cache) override; + + virtual void WaitAll( + std::vector handles) override; + + private: + struct CreateContext : public Cache::CreateContext { + const Slice* key; + bool advise_erase; + const Cache::CacheItemHelper* helper; + Cache::CreateContext* inner_ctx; + std::shared_ptr inner_handle; + SecondaryCache* comp_sec_cache; + Statistics* stats; + }; + + class ResultHandle : public SecondaryCacheResultHandle { + public: + ~ResultHandle() override {} + + bool IsReady() override { + if (inner_handle_ && inner_handle_->IsReady()) { + Complete(); + } + return ready_; + } + + void Wait() override { + inner_handle_->Wait(); + Complete(); + } + + size_t Size() override { return size_; } + + Cache::ObjectPtr Value() override { return value_; } + + void Complete() { + size_ = inner_handle_->Size(); + value_ = inner_handle_->Value(); + inner_handle_.reset(); + ready_ = true; + } + + void SetInnerHandle(std::unique_ptr&& handle) { + inner_handle_ = std::move(handle); + } + + void SetSize(size_t size) { size_ = size; } + + void SetValue(Cache::ObjectPtr val) { value_ = val; } + + CreateContext* ctx() { return &ctx_; } + + SecondaryCacheResultHandle* inner_handle() { return inner_handle_.get(); } + + private: + std::unique_ptr inner_handle_; + CreateContext ctx_; + size_t size_; + Cache::ObjectPtr value_; + bool ready_ = false; + }; + + static void NoopDelete(Cache::ObjectPtr /*obj*/, + MemoryAllocator* /*allocator*/) { + assert(false); + } + static size_t ZeroSize(Cache::ObjectPtr /*obj*/) { + assert(false); + return 0; + } + static Status NoopSaveTo(Cache::ObjectPtr /*from_obj*/, + size_t /*from_offset*/, size_t /*length*/, + char* /*out_buf*/) { + assert(false); + return Status::OK(); + } + static Status MaybeInsertAndCreate(const Slice& data, CompressionType type, + CacheTier source, + Cache::CreateContext* ctx, + MemoryAllocator* allocator, + Cache::ObjectPtr* out_obj, + size_t* out_charge); + + static const Cache::CacheItemHelper* GetHelper() { + const static Cache::CacheItemHelper basic_helper(CacheEntryRole::kMisc, + &NoopDelete); + const static Cache::CacheItemHelper maybe_insert_and_create_helper{ + CacheEntryRole::kMisc, &NoopDelete, &ZeroSize, + &NoopSaveTo, &MaybeInsertAndCreate, &basic_helper, + }; + return &maybe_insert_and_create_helper; + } + + std::shared_ptr comp_sec_cache_; + std::shared_ptr nvm_sec_cache_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/cache/tiered_secondary_cache_test.cc b/cache/tiered_secondary_cache_test.cc new file mode 100644 index 000000000..28a393325 --- /dev/null +++ b/cache/tiered_secondary_cache_test.cc @@ -0,0 +1,831 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "cache/compressed_secondary_cache.h" +#include "cache/secondary_cache_adapter.h" +#include "db/db_test_util.h" +#include "rocksdb/cache.h" +#include "rocksdb/secondary_cache.h" +#include "typed_cache.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class TestSecondaryCache : public SecondaryCache { + public: + explicit TestSecondaryCache(size_t capacity, bool ready_before_wait) + : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */, + nullptr, kDefaultToAdaptiveMutex, + kDontChargeCacheMetadata)), + ready_before_wait_(ready_before_wait), + num_insert_saved_(0), + num_hits_(0), + num_misses_(0) {} + + const char* Name() const override { return "TestSecondaryCache"; } + + Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/, + const Cache::CacheItemHelper* /*helper*/, + bool /*force_insert*/) override { + assert(false); + return Status::NotSupported(); + } + + Status InsertSaved(const Slice& key, const Slice& saved, + CompressionType type = kNoCompression, + CacheTier source = CacheTier::kVolatileTier) override { + CheckCacheKeyCommonPrefix(key); + size_t size; + char* buf; + Status s; + + num_insert_saved_++; + size = saved.size(); + buf = new char[size + sizeof(uint64_t) + 2 * sizeof(uint16_t)]; + EncodeFixed64(buf, size); + buf += sizeof(uint64_t); + EncodeFixed16(buf, type); + buf += sizeof(uint16_t); + EncodeFixed16(buf, (uint16_t)source); + buf += sizeof(uint16_t); + memcpy(buf, saved.data(), size); + buf -= sizeof(uint64_t) + 2 * sizeof(uint16_t); + if (!s.ok()) { + delete[] buf; + return s; + } + return cache_.Insert(key, buf, size); + } + + std::unique_ptr Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool wait, bool /*advise_erase*/, + Statistics* /*stats*/, bool& kept_in_sec_cache) override { + std::string key_str = key.ToString(); + TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str); + + std::unique_ptr secondary_handle; + kept_in_sec_cache = false; + + TypedHandle* handle = cache_.Lookup(key); + if (handle) { + num_hits_++; + Cache::ObjectPtr value = nullptr; + size_t charge = 0; + Status s; + char* ptr = cache_.Value(handle); + CompressionType type; + CacheTier source; + size_t size = DecodeFixed64(ptr); + ptr += sizeof(uint64_t); + type = static_cast(DecodeFixed16(ptr)); + ptr += sizeof(uint16_t); + source = static_cast(DecodeFixed16(ptr)); + assert(source == CacheTier::kVolatileTier); + ptr += sizeof(uint16_t); + s = helper->create_cb(Slice(ptr, size), type, source, create_context, + /*alloc*/ nullptr, &value, &charge); + if (s.ok()) { + secondary_handle.reset(new TestSecondaryCacheResultHandle( + cache_.get(), handle, value, charge, + /*ready=*/wait || ready_before_wait_)); + kept_in_sec_cache = true; + } else { + cache_.Release(handle); + } + } else { + num_misses_++; + } + return secondary_handle; + } + + bool SupportForceErase() const override { return false; } + + void Erase(const Slice& /*key*/) override {} + + void WaitAll(std::vector handles) override { + for (SecondaryCacheResultHandle* handle : handles) { + TestSecondaryCacheResultHandle* sec_handle = + static_cast(handle); + EXPECT_FALSE(sec_handle->IsReady()); + sec_handle->SetReady(); + } + } + + std::string GetPrintableOptions() const override { return ""; } + + uint32_t num_insert_saved() { return num_insert_saved_; } + + uint32_t num_hits() { return num_hits_; } + + uint32_t num_misses() { return num_misses_; } + + void CheckCacheKeyCommonPrefix(const Slice& key) { + Slice current_prefix(key.data(), OffsetableCacheKey::kCommonPrefixSize); + if (ckey_prefix_.empty()) { + ckey_prefix_ = current_prefix.ToString(); + } else { + EXPECT_EQ(ckey_prefix_, current_prefix.ToString()); + } + } + + private: + class TestSecondaryCacheResultHandle : public SecondaryCacheResultHandle { + public: + TestSecondaryCacheResultHandle(Cache* cache, Cache::Handle* handle, + Cache::ObjectPtr value, size_t size, + bool ready) + : cache_(cache), + handle_(handle), + value_(value), + size_(size), + is_ready_(ready) {} + + ~TestSecondaryCacheResultHandle() override { cache_->Release(handle_); } + + bool IsReady() override { return is_ready_; } + + void Wait() override {} + + Cache::ObjectPtr Value() override { + assert(is_ready_); + return value_; + } + + size_t Size() override { return Value() ? size_ : 0; } + + void SetReady() { is_ready_ = true; } + + private: + Cache* cache_; + Cache::Handle* handle_; + Cache::ObjectPtr value_; + size_t size_; + bool is_ready_; + }; + + using SharedCache = + BasicTypedSharedCacheInterface; + using TypedHandle = SharedCache::TypedHandle; + SharedCache cache_; + bool ready_before_wait_; + uint32_t num_insert_saved_; + uint32_t num_hits_; + uint32_t num_misses_; + std::string ckey_prefix_; +}; + +class DBTieredSecondaryCacheTest : public DBTestBase { + public: + DBTieredSecondaryCacheTest() + : DBTestBase("db_tiered_secondary_cache_test", /*env_do_fsync=*/true) {} + + std::shared_ptr NewCache( + size_t pri_capacity, size_t compressed_capacity, size_t nvm_capacity, + TieredAdmissionPolicy adm_policy = TieredAdmissionPolicy::kAdmPolicyAuto, + bool ready_before_wait = false) { + LRUCacheOptions lru_opts; + TieredCacheOptions opts; + lru_opts.capacity = 0; + lru_opts.num_shard_bits = 0; + lru_opts.high_pri_pool_ratio = 0; + opts.cache_opts = &lru_opts; + opts.cache_type = PrimaryCacheType::kCacheTypeLRU; + opts.comp_cache_opts.capacity = 0; + opts.comp_cache_opts.num_shard_bits = 0; + opts.total_capacity = pri_capacity + compressed_capacity; + opts.compressed_secondary_ratio = compressed_secondary_ratio_ = + (double)compressed_capacity / opts.total_capacity; + if (nvm_capacity > 0) { + nvm_sec_cache_.reset( + new TestSecondaryCache(nvm_capacity, ready_before_wait)); + opts.nvm_sec_cache = nvm_sec_cache_; + } + opts.adm_policy = adm_policy; + cache_ = NewTieredCache(opts); + assert(cache_ != nullptr); + + return cache_; + } + + void ClearPrimaryCache() { + ASSERT_EQ(UpdateTieredCache(cache_, -1, 1.0), Status::OK()); + ASSERT_EQ(UpdateTieredCache(cache_, -1, compressed_secondary_ratio_), + Status::OK()); + } + + TestSecondaryCache* nvm_sec_cache() { return nvm_sec_cache_.get(); } + + CompressedSecondaryCache* compressed_secondary_cache() { + return static_cast( + static_cast(cache_.get()) + ->TEST_GetSecondaryCache()); + } + + private: + std::shared_ptr cache_; + std::shared_ptr nvm_sec_cache_; + double compressed_secondary_ratio_; +}; + +// In this test, the block size is set to 4096. Each value is 1007 bytes, so +// each data block contains exactly 4 KV pairs. Metadata blocks are not +// cached, so we can accurately estimate the cache usage. +TEST_F(DBTieredSecondaryCacheTest, BasicTest) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + return; + } + + BlockBasedTableOptions table_options; + // We want a block cache of size 5KB, and a compressed secondary cache of + // size 5KB. However, we specify a block cache size of 256KB here in order + // to take into account the cache reservation in the block cache on + // behalf of the compressed cache. The unit of cache reservation is 256KB. + // The effective block cache capacity will be calculated as 256 + 5 = 261KB, + // and 256KB will be reserved for the compressed cache, leaving 5KB for + // the primary block cache. We only have to worry about this here because + // the cache size is so small. + table_options.block_cache = NewCache(256 * 1024, 5 * 1024, 256 * 1024); + table_options.block_size = 4 * 1024; + table_options.cache_index_and_filter_blocks = false; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + // Disable paranoid_file_checks so that flush will not read back the newly + // written file + options.paranoid_file_checks = false; + DestroyAndReopen(options); + Random rnd(301); + const int N = 256; + for (int i = 0; i < N; i++) { + std::string p_v; + test::CompressibleString(&rnd, 0.5, 1007, &p_v); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + + // The first 2 Gets, for keys 0 and 5, will load the corresponding data + // blocks as they will be cache misses. The nvm secondary cache will be + // warmed up with the compressed blocks + std::string v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 1u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 1u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + + // At this point, the nvm cache is warmed up with the data blocks for 0 + // and 5. The next Get will lookup the block in nvm and will be a hit. + // It will be created as a standalone entry in memory, and a placeholder + // will be inserted in the primary and compressed caches. + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 1u); + + // For this Get, the primary and compressed only have placeholders for + // the required data block. So we will lookup the nvm cache and find the + // block there. This time, the block will be promoted to the primary + // block cache. No promotion to the compressed secondary cache happens, + // and it will retain the placeholder. + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 2u); + + // This Get will find the data block in the primary cache. + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 2u); + + // We repeat the sequence for key 5. This will end up evicting the block + // for 0 from the in-memory cache. + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 3u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u); + + // This Get for key 0 will find the data block in nvm. Since the compressed + // cache still has the placeholder, the block (compressed) will be + // admitted. It is theh inserted into the primary as a standalone entry. + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 5u); + + // This Get for key 0 will find the data block in the compressed secondary + // cache. + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 5u); + + Destroy(options); +} + +// This test is very similar to BasicTest, except it calls MultiGet rather +// than Get, in order to exercise the async lookup and WaitAll path. +TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + return; + } + + BlockBasedTableOptions table_options; + table_options.block_cache = NewCache(260 * 1024, 10 * 1024, 256 * 1024); + table_options.block_size = 4 * 1024; + table_options.cache_index_and_filter_blocks = false; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + options.paranoid_file_checks = false; + DestroyAndReopen(options); + Random rnd(301); + const int N = 256; + for (int i = 0; i < N; i++) { + std::string p_v; + test::CompressibleString(&rnd, 0.5, 1007, &p_v); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + + std::vector keys; + std::vector values; + + keys.push_back(Key(0)); + keys.push_back(Key(4)); + keys.push_back(Key(8)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u); + + keys.clear(); + values.clear(); + keys.push_back(Key(12)); + keys.push_back(Key(16)); + keys.push_back(Key(20)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u); + + keys.clear(); + values.clear(); + keys.push_back(Key(0)); + keys.push_back(Key(4)); + keys.push_back(Key(8)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 3u); + + keys.clear(); + values.clear(); + keys.push_back(Key(0)); + keys.push_back(Key(4)); + keys.push_back(Key(8)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 6u); + + keys.clear(); + values.clear(); + keys.push_back(Key(0)); + keys.push_back(Key(4)); + keys.push_back(Key(8)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 6u); + + keys.clear(); + values.clear(); + keys.push_back(Key(12)); + keys.push_back(Key(16)); + keys.push_back(Key(20)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 9u); + + keys.clear(); + values.clear(); + keys.push_back(Key(12)); + keys.push_back(Key(16)); + keys.push_back(Key(20)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 12u); + + keys.clear(); + values.clear(); + keys.push_back(Key(12)); + keys.push_back(Key(16)); + keys.push_back(Key(20)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 12u); + + Destroy(options); +} + +TEST_F(DBTieredSecondaryCacheTest, WaitAllTest) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + return; + } + + BlockBasedTableOptions table_options; + table_options.block_cache = NewCache(250 * 1024, 20 * 1024, 256 * 1024); + table_options.block_size = 4 * 1024; + table_options.cache_index_and_filter_blocks = false; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + options.paranoid_file_checks = false; + DestroyAndReopen(options); + Random rnd(301); + const int N = 256; + for (int i = 0; i < N; i++) { + std::string p_v; + test::CompressibleString(&rnd, 0.5, 1007, &p_v); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + + std::vector keys; + std::vector values; + + keys.push_back(Key(0)); + keys.push_back(Key(4)); + keys.push_back(Key(8)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u); + + keys.clear(); + values.clear(); + keys.push_back(Key(12)); + keys.push_back(Key(16)); + keys.push_back(Key(20)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u); + + // Insert placeholders for 4 in primary and compressed + std::string val = Get(Key(4)); + + // Force placeholder 4 out of primary + keys.clear(); + values.clear(); + keys.push_back(Key(24)); + keys.push_back(Key(28)); + keys.push_back(Key(32)); + keys.push_back(Key(36)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 10u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 10u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 1u); + + // Now read 4 again. This will create a placeholder in primary, and insert + // in compressed secondary since it already has a placeholder + val = Get(Key(4)); + + // Now read 0, 4 and 8. While 4 is already in the compressed secondary + // cache, 0 and 8 will be read asynchronously from the nvm tier. The + // WaitAll will be called for all 3 blocks. + keys.clear(); + values.clear(); + keys.push_back(Key(0)); + keys.push_back(Key(4)); + keys.push_back(Key(8)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 10u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 10u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u); + + Destroy(options); +} + +TEST_F(DBTieredSecondaryCacheTest, ReadyBeforeWaitAllTest) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + return; + } + + BlockBasedTableOptions table_options; + table_options.block_cache = NewCache(250 * 1024, 20 * 1024, 256 * 1024, + TieredAdmissionPolicy::kAdmPolicyAuto, + /*ready_before_wait=*/true); + table_options.block_size = 4 * 1024; + table_options.cache_index_and_filter_blocks = false; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.statistics = CreateDBStatistics(); + + options.paranoid_file_checks = false; + DestroyAndReopen(options); + Random rnd(301); + const int N = 256; + for (int i = 0; i < N; i++) { + std::string p_v; + test::CompressibleString(&rnd, 0.5, 1007, &p_v); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + + std::vector keys; + std::vector values; + + keys.push_back(Key(0)); + keys.push_back(Key(4)); + keys.push_back(Key(8)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u); + ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 3u); + + keys.clear(); + values.clear(); + keys.push_back(Key(12)); + keys.push_back(Key(16)); + keys.push_back(Key(20)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u); + ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 6u); + + keys.clear(); + values.clear(); + keys.push_back(Key(0)); + keys.push_back(Key(4)); + keys.push_back(Key(8)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 3u); + ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 6u); + + ClearPrimaryCache(); + + keys.clear(); + values.clear(); + keys.push_back(Key(0)); + keys.push_back(Key(32)); + keys.push_back(Key(36)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u); + ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 8u); + + keys.clear(); + values.clear(); + keys.push_back(Key(0)); + keys.push_back(Key(32)); + keys.push_back(Key(36)); + values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true); + ASSERT_EQ(values.size(), keys.size()); + for (auto value : values) { + ASSERT_EQ(1007, value.size()); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u); + ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 8u); + + Destroy(options); +} + +// This test is for iteration. It iterates through a set of keys in two +// passes. First pass loads the compressed blocks into the nvm tier, and +// the second pass should hit all of those blocks. +TEST_F(DBTieredSecondaryCacheTest, IterateTest) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + return; + } + + BlockBasedTableOptions table_options; + table_options.block_cache = NewCache(250 * 1024, 10 * 1024, 256 * 1024); + table_options.block_size = 4 * 1024; + table_options.cache_index_and_filter_blocks = false; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + options.paranoid_file_checks = false; + DestroyAndReopen(options); + Random rnd(301); + const int N = 256; + for (int i = 0; i < N; i++) { + std::string p_v; + test::CompressibleString(&rnd, 0.5, 1007, &p_v); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + + ReadOptions ro; + ro.readahead_size = 256 * 1024; + auto iter = dbfull()->NewIterator(ro); + iter->SeekToFirst(); + for (int i = 0; i < 31; ++i) { + ASSERT_EQ(Key(i), iter->key().ToString()); + ASSERT_EQ(1007, iter->value().size()); + iter->Next(); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u); + delete iter; + + iter = dbfull()->NewIterator(ro); + iter->SeekToFirst(); + for (int i = 0; i < 31; ++i) { + ASSERT_EQ(Key(i), iter->key().ToString()); + ASSERT_EQ(1007, iter->value().size()); + iter->Next(); + } + ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u); + ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u); + ASSERT_EQ(nvm_sec_cache()->num_hits(), 8u); + delete iter; + + Destroy(options); +} + +class DBTieredAdmPolicyTest + : public DBTieredSecondaryCacheTest, + public testing::WithParamInterface {}; + +TEST_P(DBTieredAdmPolicyTest, CompressedOnlyTest) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + return; + } + + BlockBasedTableOptions table_options; + // We want a block cache of size 10KB, and a compressed secondary cache of + // size 10KB. However, we specify a block cache size of 256KB here in order + // to take into account the cache reservation in the block cache on + // behalf of the compressed cache. The unit of cache reservation is 256KB. + // The effective block cache capacity will be calculated as 256 + 10 = 266KB, + // and 256KB will be reserved for the compressed cache, leaving 10KB for + // the primary block cache. We only have to worry about this here because + // the cache size is so small. + table_options.block_cache = NewCache(256 * 1024, 10 * 1024, 0, GetParam()); + table_options.block_size = 4 * 1024; + table_options.cache_index_and_filter_blocks = false; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + size_t comp_cache_usage = compressed_secondary_cache()->TEST_GetUsage(); + // Disable paranoid_file_checks so that flush will not read back the newly + // written file + options.paranoid_file_checks = false; + DestroyAndReopen(options); + Random rnd(301); + const int N = 256; + for (int i = 0; i < N; i++) { + std::string p_v; + test::CompressibleString(&rnd, 0.5, 1007, &p_v); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + + // The first 2 Gets, for keys 0 and 5, will load the corresponding data + // blocks as they will be cache misses. Since this is a 2-tier cache ( + // primary and compressed), no warm-up should happen with the compressed + // blocks. + std::string v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + + ASSERT_EQ(compressed_secondary_cache()->TEST_GetUsage(), comp_cache_usage); + + Destroy(options); +} + +INSTANTIATE_TEST_CASE_P( + DBTieredAdmPolicyTest, DBTieredAdmPolicyTest, + ::testing::Values(TieredAdmissionPolicy::kAdmPolicyAuto, + TieredAdmissionPolicy::kAdmPolicyPlaceholder, + TieredAdmissionPolicy::kAdmPolicyAllowCacheHits)); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/cache/typed_cache.h b/cache/typed_cache.h index e42aa4c26..125bfa0f5 100644 --- a/cache/typed_cache.h +++ b/cache/typed_cache.h @@ -234,15 +234,19 @@ class FullTypedCacheHelperFns : public BasicTypedCacheHelperFns { return Status::OK(); } - static Status Create(const Slice& data, CreateContext* context, + static Status Create(const Slice& data, CompressionType type, + CacheTier source, CreateContext* context, MemoryAllocator* allocator, ObjectPtr* out_obj, size_t* out_charge) { std::unique_ptr value = nullptr; + if (source != CacheTier::kVolatileTier) { + return Status::InvalidArgument(); + } if constexpr (sizeof(TCreateContext) > 0) { TCreateContext* tcontext = static_cast(context); - tcontext->Create(&value, out_charge, data, allocator); + tcontext->Create(&value, out_charge, data, type, allocator); } else { - TCreateContext::Create(&value, out_charge, data, allocator); + TCreateContext::Create(&value, out_charge, data, type, allocator); } *out_obj = UpCastValue(value.release()); return Status::OK(); @@ -301,13 +305,15 @@ class FullTypedCacheInterface inline Status InsertFull( const Slice& key, TValuePtr value, size_t charge, TypedHandle** handle = nullptr, Priority priority = Priority::LOW, - CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) { + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier, + const Slice& compressed = Slice(), + CompressionType type = CompressionType::kNoCompression) { auto untyped_handle = reinterpret_cast(handle); - auto helper = lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier + auto helper = lowest_used_cache_tier > CacheTier::kVolatileTier ? GetFullHelper() : GetBasicHelper(); return this->cache_->Insert(key, UpCastValue(value), helper, charge, - untyped_handle, priority); + untyped_handle, priority, compressed, type); } // Like SecondaryCache::InsertSaved, with SecondaryCache compatibility @@ -319,9 +325,9 @@ class FullTypedCacheInterface size_t* out_charge = nullptr) { ObjectPtr value; size_t charge; - Status st = GetFullHelper()->create_cb(data, create_context, - this->cache_->memory_allocator(), - &value, &charge); + Status st = GetFullHelper()->create_cb( + data, kNoCompression, CacheTier::kVolatileTier, create_context, + this->cache_->memory_allocator(), &value, &charge); if (out_charge) { *out_charge = charge; } @@ -340,7 +346,7 @@ class FullTypedCacheInterface const Slice& key, TCreateContext* create_context = nullptr, Priority priority = Priority::LOW, Statistics* stats = nullptr, CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) { - if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) { + if (lowest_used_cache_tier > CacheTier::kVolatileTier) { return reinterpret_cast(this->cache_->Lookup( key, GetFullHelper(), create_context, priority, stats)); } else { @@ -352,7 +358,7 @@ class FullTypedCacheInterface inline void StartAsyncLookupFull( TypedAsyncLookupHandle& async_handle, CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) { - if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) { + if (lowest_used_cache_tier > CacheTier::kVolatileTier) { async_handle.helper = GetFullHelper(); this->cache_->StartAsyncLookup(async_handle); } else { diff --git a/db/arena_wrapped_db_iter.cc b/db/arena_wrapped_db_iter.cc index b101fbbc7..e6dcb6696 100644 --- a/db/arena_wrapped_db_iter.cc +++ b/db/arena_wrapped_db_iter.cc @@ -19,6 +19,14 @@ namespace ROCKSDB_NAMESPACE { +inline static SequenceNumber GetSeqNum(const DBImpl* db, const Snapshot* s) { + if (s) { + return s->GetSequenceNumber(); + } else { + return db->GetLatestSequenceNumber(); + } +} + Status ArenaWrappedDBIter::GetProperty(std::string prop_name, std::string* prop) { if (prop_name == "rocksdb.iterator.super-version-number") { @@ -54,7 +62,9 @@ void ArenaWrappedDBIter::Init( } } -Status ArenaWrappedDBIter::Refresh() { +Status ArenaWrappedDBIter::Refresh() { return Refresh(nullptr); } + +Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) { if (cfd_ == nullptr || db_impl_ == nullptr || !allow_refresh_) { return Status::NotSupported("Creating renew iterator is not allowed."); } @@ -63,6 +73,10 @@ Status ArenaWrappedDBIter::Refresh() { // correct behavior. Will be corrected automatically when we take a snapshot // here for the case of WritePreparedTxnDB. uint64_t cur_sv_number = cfd_->GetSuperVersionNumber(); + // If we recreate a new internal iterator below (NewInternalIterator()), + // we will pass in read_options_. We need to make sure it + // has the right snapshot. + read_options_.snapshot = snapshot; TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1"); TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2"); auto reinit_internal_iter = [&]() { @@ -72,18 +86,19 @@ Status ArenaWrappedDBIter::Refresh() { new (&arena_) Arena(); SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_); - SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber(); + assert(sv->version_number >= cur_sv_number); + SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot); if (read_callback_) { - read_callback_->Refresh(latest_seq); + read_callback_->Refresh(read_seq); } Init(env, read_options_, *(cfd_->ioptions()), sv->mutable_cf_options, - sv->current, latest_seq, + sv->current, read_seq, sv->mutable_cf_options.max_sequential_skip_in_iterations, - cur_sv_number, read_callback_, db_impl_, cfd_, expose_blob_index_, + sv->version_number, read_callback_, db_impl_, cfd_, expose_blob_index_, allow_refresh_); InternalIterator* internal_iter = db_impl_->NewInternalIterator( - read_options_, cfd_, sv, &arena_, latest_seq, + read_options_, cfd_, sv, &arena_, read_seq, /* allow_unprepared_value */ true, /* db_iter */ this); SetIterUnderDBIter(internal_iter); }; @@ -92,13 +107,13 @@ Status ArenaWrappedDBIter::Refresh() { reinit_internal_iter(); break; } else { - SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber(); + SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot); // Refresh range-tombstones in MemTable if (!read_options_.ignore_range_deletions) { SuperVersion* sv = cfd_->GetThreadLocalSuperVersion(db_impl_); TEST_SYNC_POINT_CALLBACK("ArenaWrappedDBIter::Refresh:SV", nullptr); auto t = sv->mem->NewRangeTombstoneIterator( - read_options_, latest_seq, false /* immutable_memtable */); + read_options_, read_seq, false /* immutable_memtable */); if (!t || t->empty()) { // If memtable_range_tombstone_iter_ points to a non-empty tombstone // iterator, then it means sv->mem is not the memtable that @@ -128,9 +143,6 @@ Status ArenaWrappedDBIter::Refresh() { } db_impl_->ReturnAndCleanupSuperVersion(cfd_, sv); } - // Refresh latest sequence number - db_iter_->set_sequence(latest_seq); - db_iter_->set_valid(false); // Check again if the latest super version number is changed uint64_t latest_sv_number = cfd_->GetSuperVersionNumber(); if (latest_sv_number != cur_sv_number) { @@ -139,6 +151,8 @@ Status ArenaWrappedDBIter::Refresh() { cur_sv_number = latest_sv_number; continue; } + db_iter_->set_sequence(read_seq); + db_iter_->set_valid(false); break; } } diff --git a/db/arena_wrapped_db_iter.h b/db/arena_wrapped_db_iter.h index f15be306d..678ea3e78 100644 --- a/db/arena_wrapped_db_iter.h +++ b/db/arena_wrapped_db_iter.h @@ -80,6 +80,7 @@ class ArenaWrappedDBIter : public Iterator { Status GetProperty(std::string prop_name, std::string* prop) override; Status Refresh() override; + Status Refresh(const Snapshot*) override; void Init(Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions, diff --git a/db/blob/blob_contents.h b/db/blob/blob_contents.h index 15a672a0a..40b94d51f 100644 --- a/db/blob/blob_contents.h +++ b/db/blob/blob_contents.h @@ -46,7 +46,8 @@ class BlobContents { class BlobContentsCreator : public Cache::CreateContext { public: static void Create(std::unique_ptr* out, size_t* out_charge, - const Slice& contents, MemoryAllocator* alloc) { + const Slice& contents, CompressionType /*type*/, + MemoryAllocator* alloc) { auto raw = new BlobContents(AllocateAndCopyBlock(contents, alloc), contents.size()); out->reset(raw); diff --git a/db/blob/blob_file_builder.cc b/db/blob/blob_file_builder.cc index 21c1e5d41..35269fdb5 100644 --- a/db/blob/blob_file_builder.cc +++ b/db/blob/blob_file_builder.cc @@ -261,7 +261,7 @@ Status BlobFileBuilder::CompressBlobIfNeeded( // TODO: allow user CompressionOptions, including max_compressed_bytes_per_kb CompressionOptions opts; - CompressionContext context(blob_compression_type_); + CompressionContext context(blob_compression_type_, opts); constexpr uint64_t sample_for_compression = 0; CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), diff --git a/db/blob/blob_file_builder_test.cc b/db/blob/blob_file_builder_test.cc index 3a0feee45..5882e219f 100644 --- a/db/blob/blob_file_builder_test.cc +++ b/db/blob/blob_file_builder_test.cc @@ -406,7 +406,7 @@ TEST_F(BlobFileBuilderTest, Compression) { ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1); CompressionOptions opts; - CompressionContext context(kSnappyCompression); + CompressionContext context(kSnappyCompression, opts); constexpr uint64_t sample_for_compression = 0; CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), diff --git a/db/blob/blob_file_reader.cc b/db/blob/blob_file_reader.cc index 6df7f3aee..0c30efbc1 100644 --- a/db/blob/blob_file_reader.cc +++ b/db/blob/blob_file_reader.cc @@ -154,11 +154,9 @@ Status BlobFileReader::ReadHeader(const RandomAccessFileReader* file_reader, constexpr uint64_t read_offset = 0; constexpr size_t read_size = BlobLogHeader::kSize; - // TODO: rate limit reading headers from blob files. const Status s = ReadFromFile(file_reader, read_options, read_offset, read_size, - statistics, &header_slice, &buf, &aligned_buf, - Env::IO_TOTAL /* rate_limiter_priority */); + statistics, &header_slice, &buf, &aligned_buf); if (!s.ok()) { return s; } @@ -207,11 +205,9 @@ Status BlobFileReader::ReadFooter(const RandomAccessFileReader* file_reader, const uint64_t read_offset = file_size - BlobLogFooter::kSize; constexpr size_t read_size = BlobLogFooter::kSize; - // TODO: rate limit reading footers from blob files. const Status s = ReadFromFile(file_reader, read_options, read_offset, read_size, - statistics, &footer_slice, &buf, &aligned_buf, - Env::IO_TOTAL /* rate_limiter_priority */); + statistics, &footer_slice, &buf, &aligned_buf); if (!s.ok()) { return s; } @@ -242,8 +238,7 @@ Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader, const ReadOptions& read_options, uint64_t read_offset, size_t read_size, Statistics* statistics, Slice* slice, - Buffer* buf, AlignedBuf* aligned_buf, - Env::IOPriority rate_limiter_priority) { + Buffer* buf, AlignedBuf* aligned_buf) { assert(slice); assert(buf); assert(aligned_buf); @@ -264,13 +259,13 @@ Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader, constexpr char* scratch = nullptr; s = file_reader->Read(io_options, read_offset, read_size, slice, scratch, - aligned_buf, rate_limiter_priority); + aligned_buf); } else { buf->reset(new char[read_size]); constexpr AlignedBuf* aligned_scratch = nullptr; s = file_reader->Read(io_options, read_offset, read_size, slice, buf->get(), - aligned_scratch, rate_limiter_priority); + aligned_scratch); } if (!s.ok()) { @@ -345,8 +340,7 @@ Status BlobFileReader::GetBlob( } prefetched = prefetch_buffer->TryReadFromCache( io_options, file_reader_.get(), record_offset, - static_cast(record_size), &record_slice, &s, - read_options.rate_limiter_priority, for_compaction); + static_cast(record_size), &record_slice, &s, for_compaction); if (!s.ok()) { return s; } @@ -357,10 +351,10 @@ Status BlobFileReader::GetBlob( PERF_COUNTER_ADD(blob_read_count, 1); PERF_COUNTER_ADD(blob_read_byte, record_size); PERF_TIMER_GUARD(blob_read_time); - const Status s = ReadFromFile( - file_reader_.get(), read_options, record_offset, - static_cast(record_size), statistics_, &record_slice, &buf, - &aligned_buf, read_options.rate_limiter_priority); + const Status s = + ReadFromFile(file_reader_.get(), read_options, record_offset, + static_cast(record_size), statistics_, + &record_slice, &buf, &aligned_buf); if (!s.ok()) { return s; } @@ -468,9 +462,12 @@ void BlobFileReader::MultiGetBlob( TEST_SYNC_POINT("BlobFileReader::MultiGetBlob:ReadFromFile"); PERF_COUNTER_ADD(blob_read_count, num_blobs); PERF_COUNTER_ADD(blob_read_byte, total_len); - s = file_reader_->MultiRead(IOOptions(), read_reqs.data(), read_reqs.size(), - direct_io ? &aligned_buf : nullptr, - read_options.rate_limiter_priority); + IOOptions opts; + s = file_reader_->PrepareIOOptions(read_options, opts); + if (s.ok()) { + s = file_reader_->MultiRead(opts, read_reqs.data(), read_reqs.size(), + direct_io ? &aligned_buf : nullptr); + } if (!s.ok()) { for (auto& req : read_reqs) { req.status.PermitUncheckedError(); @@ -588,7 +585,8 @@ Status BlobFileReader::UncompressBlobIfNeeded( assert(result); if (compression_type == kNoCompression) { - BlobContentsCreator::Create(result, nullptr, value_slice, allocator); + BlobContentsCreator::Create(result, nullptr, value_slice, kNoCompression, + allocator); return Status::OK(); } diff --git a/db/blob/blob_file_reader.h b/db/blob/blob_file_reader.h index 990e32540..fa8aa501d 100644 --- a/db/blob/blob_file_reader.h +++ b/db/blob/blob_file_reader.h @@ -89,8 +89,7 @@ class BlobFileReader { const ReadOptions& read_options, uint64_t read_offset, size_t read_size, Statistics* statistics, Slice* slice, Buffer* buf, - AlignedBuf* aligned_buf, - Env::IOPriority rate_limiter_priority); + AlignedBuf* aligned_buf); static Status VerifyBlob(const Slice& record_slice, const Slice& user_key, uint64_t value_size); diff --git a/db/blob/blob_file_reader_test.cc b/db/blob/blob_file_reader_test.cc index c8e4e5954..b6049d1ef 100644 --- a/db/blob/blob_file_reader_test.cc +++ b/db/blob/blob_file_reader_test.cc @@ -74,7 +74,7 @@ void WriteBlobFile(const ImmutableOptions& immutable_options, } } else { CompressionOptions opts; - CompressionContext context(compression); + CompressionContext context(compression, opts); constexpr uint64_t sample_for_compression = 0; CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), compression, sample_for_compression); diff --git a/db/blob/blob_log_sequential_reader.cc b/db/blob/blob_log_sequential_reader.cc index 2ed430681..579c98e29 100644 --- a/db/blob/blob_log_sequential_reader.cc +++ b/db/blob/blob_log_sequential_reader.cc @@ -29,9 +29,8 @@ Status BlobLogSequentialReader::ReadSlice(uint64_t size, Slice* slice, StopWatch read_sw(clock_, statistics_, BLOB_DB_BLOB_FILE_READ_MICROS); // TODO: rate limit `BlobLogSequentialReader` reads (it appears unused?) - Status s = - file_->Read(IOOptions(), next_byte_, static_cast(size), slice, - buf, nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + Status s = file_->Read(IOOptions(), next_byte_, static_cast(size), + slice, buf, nullptr); next_byte_ += size; if (!s.ok()) { return s; diff --git a/db/blob/blob_source_test.cc b/db/blob/blob_source_test.cc index a9771565a..258d2da5e 100644 --- a/db/blob/blob_source_test.cc +++ b/db/blob/blob_source_test.cc @@ -76,7 +76,7 @@ void WriteBlobFile(const ImmutableOptions& immutable_options, } } else { CompressionOptions opts; - CompressionContext context(compression); + CompressionContext context(compression, opts); constexpr uint64_t sample_for_compression = 0; CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), compression, sample_for_compression); @@ -1220,7 +1220,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { auto sec_handle0 = secondary_cache->Lookup( key0, BlobSource::SharedCacheInterface::GetFullHelper(), /*context*/ nullptr, true, - /*advise_erase=*/true, kept_in_sec_cache); + /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache); ASSERT_FALSE(kept_in_sec_cache); ASSERT_NE(sec_handle0, nullptr); ASSERT_TRUE(sec_handle0->IsReady()); @@ -1248,7 +1248,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { auto sec_handle1 = secondary_cache->Lookup( key1, BlobSource::SharedCacheInterface::GetFullHelper(), /*context*/ nullptr, true, - /*advise_erase=*/true, kept_in_sec_cache); + /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache); ASSERT_FALSE(kept_in_sec_cache); ASSERT_EQ(sec_handle1, nullptr); diff --git a/db/blob/db_blob_basic_test.cc b/db/blob/db_blob_basic_test.cc index c6c6d2b93..1c0caba93 100644 --- a/db/blob/db_blob_basic_test.cc +++ b/db/blob/db_blob_basic_test.cc @@ -168,6 +168,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) { ASSERT_EQ(iter->value().ToString(), blobs[i]); ++i; } + ASSERT_OK(iter->status()); ASSERT_EQ(i, num_blobs); ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0); } @@ -203,6 +204,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) { ASSERT_EQ(iter->value().ToString(), blobs[i]); ++i; } + ASSERT_OK(iter->status()); ASSERT_EQ(i, num_blobs); ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), num_blobs); @@ -224,6 +226,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) { ASSERT_EQ(iter->value().ToString(), blobs[i]); ++i; } + ASSERT_OK(iter->status()); ASSERT_EQ(i, num_blobs); ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0); } @@ -2123,6 +2126,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) { /*key_is_internal*/ false); iter->Prev(); } + ASSERT_OK(iter->status()); } // Backward iteration, then reverse to forward. @@ -2169,6 +2173,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) { iter->Next(); } } + ASSERT_OK(iter->status()); } // Backward iterating multiple versions of the same key, get in this order: @@ -2187,6 +2192,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) { iter->Prev(); } } + ASSERT_OK(iter->status()); } int upper_bound_idx = num_blobs - 2; @@ -2209,6 +2215,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) { iter->Next(); } } + ASSERT_OK(iter->status()); } // Backward iteration with upper and lower bound. @@ -2224,6 +2231,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) { iter->Prev(); } } + ASSERT_OK(iter->status()); } } diff --git a/db/blob/db_blob_index_test.cc b/db/blob/db_blob_index_test.cc index eabca1358..e29976034 100644 --- a/db/blob/db_blob_index_test.cc +++ b/db/blob/db_blob_index_test.cc @@ -96,9 +96,13 @@ class DBBlobIndexTest : public DBTestBase { } ArenaWrappedDBIter* GetBlobIterator() { - return dbfull()->NewIteratorImpl( - ReadOptions(), cfd(), dbfull()->GetLatestSequenceNumber(), - nullptr /*read_callback*/, true /*expose_blob_index*/); + ColumnFamilyData* column_family = cfd(); + DBImpl* db_impl = dbfull(); + return db_impl->NewIteratorImpl( + ReadOptions(), column_family, + column_family->GetReferencedSuperVersion(db_impl), + db_impl->GetLatestSequenceNumber(), nullptr /*read_callback*/, + true /*expose_blob_index*/); } Options GetTestOptions() { diff --git a/db/builder.cc b/db/builder.cc index 84d2396c3..d3040ee9e 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -203,6 +203,7 @@ Status BuildTable( blob_file_builder.get(), ioptions.allow_data_in_errors, ioptions.enforce_single_del_contracts, /*manual_compaction_canceled=*/kManualCompactionCanceledFalse, + true /* must_count_input_entries */, /*compaction=*/nullptr, compaction_filter.get(), /*shutting_down=*/nullptr, db_options.info_log, full_history_ts_low); @@ -286,18 +287,19 @@ Status BuildTable( TEST_SYNC_POINT("BuildTable:BeforeFinishBuildTable"); const bool empty = builder->IsEmpty(); if (num_input_entries != nullptr) { + assert(c_iter.HasNumInputEntryScanned()); *num_input_entries = - c_iter.num_input_entry_scanned() + num_unfragmented_tombstones; + c_iter.NumInputEntryScanned() + num_unfragmented_tombstones; } if (!s.ok() || empty) { builder->Abandon(); } else { - std::string seqno_time_mapping_str; + std::string seqno_to_time_mapping_str; seqno_to_time_mapping.Encode( - seqno_time_mapping_str, meta->fd.smallest_seqno, + seqno_to_time_mapping_str, meta->fd.smallest_seqno, meta->fd.largest_seqno, meta->file_creation_time); builder->SetSeqnoTimeTableProperties( - seqno_time_mapping_str, + seqno_to_time_mapping_str, ioptions.compaction_style == CompactionStyle::kCompactionStyleFIFO ? meta->file_creation_time : meta->oldest_ancester_time); diff --git a/db/c.cc b/db/c.cc index 2ffa714ce..5555ae198 100644 --- a/db/c.cc +++ b/db/c.cc @@ -45,6 +45,7 @@ #include "rocksdb/utilities/transaction_db.h" #include "rocksdb/utilities/write_batch_with_index.h" #include "rocksdb/write_batch.h" +#include "rocksdb/write_buffer_manager.h" #include "utilities/merge_operators.h" using ROCKSDB_NAMESPACE::BackupEngine; @@ -77,6 +78,7 @@ using ROCKSDB_NAMESPACE::EnvOptions; using ROCKSDB_NAMESPACE::FileLock; using ROCKSDB_NAMESPACE::FilterPolicy; using ROCKSDB_NAMESPACE::FlushOptions; +using ROCKSDB_NAMESPACE::HistogramData; using ROCKSDB_NAMESPACE::HyperClockCacheOptions; using ROCKSDB_NAMESPACE::InfoLogLevel; using ROCKSDB_NAMESPACE::IngestExternalFileOptions; @@ -119,10 +121,12 @@ using ROCKSDB_NAMESPACE::TransactionDB; using ROCKSDB_NAMESPACE::TransactionDBOptions; using ROCKSDB_NAMESPACE::TransactionLogIterator; using ROCKSDB_NAMESPACE::TransactionOptions; +using ROCKSDB_NAMESPACE::WaitForCompactOptions; using ROCKSDB_NAMESPACE::WALRecoveryMode; using ROCKSDB_NAMESPACE::WritableFile; using ROCKSDB_NAMESPACE::WriteBatch; using ROCKSDB_NAMESPACE::WriteBatchWithIndex; +using ROCKSDB_NAMESPACE::WriteBufferManager; using ROCKSDB_NAMESPACE::WriteOptions; using std::unordered_set; @@ -217,6 +221,9 @@ struct rocksdb_memory_allocator_t { struct rocksdb_cache_t { std::shared_ptr rep; }; +struct rocksdb_write_buffer_manager_t { + std::shared_ptr rep; +}; struct rocksdb_livefiles_t { std::vector rep; }; @@ -274,11 +281,19 @@ struct rocksdb_optimistictransactiondb_t { struct rocksdb_optimistictransaction_options_t { OptimisticTransactionOptions rep; }; +struct rocksdb_wait_for_compact_options_t { + WaitForCompactOptions rep; +}; struct rocksdb_compactionfiltercontext_t { CompactionFilter::Context rep; }; +struct rocksdb_statistics_histogram_data_t { + rocksdb_statistics_histogram_data_t() : rep() {} + HistogramData rep; +}; + struct rocksdb_compactionfilter_t : public CompactionFilter { void* state_; void (*destructor_)(void*); @@ -2894,6 +2909,16 @@ void rocksdb_options_set_db_paths(rocksdb_options_t* opt, opt->rep.db_paths = db_paths; } +void rocksdb_options_set_cf_paths(rocksdb_options_t* opt, + const rocksdb_dbpath_t** dbpath_values, + size_t num_paths) { + std::vector cf_paths(num_paths); + for (size_t i = 0; i < num_paths; ++i) { + cf_paths[i] = dbpath_values[i]->rep; + } + opt->rep.cf_paths = cf_paths; +} + void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) { opt->rep.env = (env ? env->rep : nullptr); } @@ -2925,6 +2950,11 @@ void rocksdb_options_set_write_buffer_size(rocksdb_options_t* opt, size_t s) { opt->rep.write_buffer_size = s; } +void rocksdb_options_set_write_buffer_manager( + rocksdb_options_t* opt, rocksdb_write_buffer_manager_t* wbm) { + opt->rep.write_buffer_manager = wbm->rep; +} + size_t rocksdb_options_get_write_buffer_size(rocksdb_options_t* opt) { return opt->rep.write_buffer_size; } @@ -3019,10 +3049,43 @@ void rocksdb_options_set_max_bytes_for_level_multiplier_additional( } } +void rocksdb_options_set_periodic_compaction_seconds(rocksdb_options_t* opt, + uint64_t seconds) { + opt->rep.periodic_compaction_seconds = seconds; +} + +uint64_t rocksdb_options_get_periodic_compaction_seconds( + rocksdb_options_t* opt) { + return opt->rep.periodic_compaction_seconds; +} + void rocksdb_options_enable_statistics(rocksdb_options_t* opt) { opt->rep.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); } +void rocksdb_options_set_statistics_level(rocksdb_options_t* opt, int level) { + if (!opt->rep.statistics) { + return; + } + + if (level < rocksdb_statistics_level_disable_all) { + level = rocksdb_statistics_level_disable_all; + } + if (level > rocksdb_statistics_level_all) { + level = rocksdb_statistics_level_all; + } + opt->rep.statistics->set_stats_level( + static_cast(level)); +} + +int rocksdb_options_get_statistics_level(rocksdb_options_t* opt) { + if (!opt->rep.statistics) { + return ROCKSDB_NAMESPACE::StatsLevel::kDisableAll; + } + + return static_cast(opt->rep.statistics->get_stats_level()); +} + void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, unsigned char val) { opt->rep.skip_stats_update_on_db_open = val; @@ -3862,6 +3925,26 @@ char* rocksdb_options_statistics_get_string(rocksdb_options_t* opt) { return nullptr; } +uint64_t rocksdb_options_statistics_get_ticker_count(rocksdb_options_t* opt, + uint32_t ticker_type) { + ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get(); + if (statistics) { + return statistics->getTickerCount(ticker_type); + } + return 0; +} + +void rocksdb_options_statistics_get_histogram_data( + rocksdb_options_t* opt, uint32_t type, + rocksdb_statistics_histogram_data_t* const data) { + ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get(); + if (statistics) { + statistics->histogramData(type, &data->rep); + } else { + *data = rocksdb_statistics_histogram_data_t{}; + } +} + void rocksdb_options_set_ratelimiter(rocksdb_options_t* opt, rocksdb_ratelimiter_t* limiter) { if (limiter) { @@ -3904,6 +3987,16 @@ rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(int64_t rate_bytes_per_sec, return rate_limiter; } +rocksdb_ratelimiter_t* rocksdb_ratelimiter_create_auto_tuned( + int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness) { + rocksdb_ratelimiter_t* rate_limiter = new rocksdb_ratelimiter_t; + rate_limiter->rep.reset(NewGenericRateLimiter(rate_bytes_per_sec, + refill_period_us, fairness, + RateLimiter::Mode::kWritesOnly, + true)); // auto_tuned + return rate_limiter; +} + void rocksdb_ratelimiter_destroy(rocksdb_ratelimiter_t* limiter) { delete limiter; } @@ -4536,6 +4629,11 @@ void rocksdb_readoptions_set_iter_start_ts(rocksdb_readoptions_t* opt, } } +void rocksdb_readoptions_set_auto_readahead_size(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.auto_readahead_size = v; +} + rocksdb_writeoptions_t* rocksdb_writeoptions_create() { return new rocksdb_writeoptions_t; } @@ -4808,6 +4906,60 @@ size_t rocksdb_cache_get_occupancy_count(const rocksdb_cache_t* cache) { return cache->rep->GetOccupancyCount(); } +rocksdb_write_buffer_manager_t* rocksdb_write_buffer_manager_create( + size_t buffer_size, bool allow_stall) { + rocksdb_write_buffer_manager_t* wbm = new rocksdb_write_buffer_manager_t; + wbm->rep.reset(new WriteBufferManager(buffer_size, {}, allow_stall)); + return wbm; +} + +rocksdb_write_buffer_manager_t* rocksdb_write_buffer_manager_create_with_cache( + size_t buffer_size, const rocksdb_cache_t* cache, bool allow_stall) { + rocksdb_write_buffer_manager_t* wbm = new rocksdb_write_buffer_manager_t; + wbm->rep.reset(new WriteBufferManager(buffer_size, cache->rep, allow_stall)); + return wbm; +} + +void rocksdb_write_buffer_manager_destroy(rocksdb_write_buffer_manager_t* wbm) { + delete wbm; +} + +bool rocksdb_write_buffer_manager_enabled(rocksdb_write_buffer_manager_t* wbm) { + return wbm->rep->enabled(); +} + +bool rocksdb_write_buffer_manager_cost_to_cache( + rocksdb_write_buffer_manager_t* wbm) { + return wbm->rep->cost_to_cache(); +} + +size_t rocksdb_write_buffer_manager_memory_usage( + rocksdb_write_buffer_manager_t* wbm) { + return wbm->rep->memory_usage(); +} + +size_t rocksdb_write_buffer_manager_mutable_memtable_memory_usage( + rocksdb_write_buffer_manager_t* wbm) { + return wbm->rep->mutable_memtable_memory_usage(); +} + +size_t rocksdb_write_buffer_manager_dummy_entries_in_cache_usage( + rocksdb_write_buffer_manager_t* wbm) { + return wbm->rep->dummy_entries_in_cache_usage(); +} +size_t rocksdb_write_buffer_manager_buffer_size( + rocksdb_write_buffer_manager_t* wbm) { + return wbm->rep->buffer_size(); +} +void rocksdb_write_buffer_manager_set_buffer_size( + rocksdb_write_buffer_manager_t* wbm, size_t new_size) { + wbm->rep->SetBufferSize(new_size); +} +ROCKSDB_LIBRARY_API void rocksdb_write_buffer_manager_set_allow_stall( + rocksdb_write_buffer_manager_t* wbm, bool new_allow_stall) { + wbm->rep->SetAllowStall(new_allow_stall); +} + rocksdb_dbpath_t* rocksdb_dbpath_create(const char* path, uint64_t target_size) { rocksdb_dbpath_t* result = new rocksdb_dbpath_t; @@ -5194,7 +5346,8 @@ rocksdb_fifo_compaction_options_t* rocksdb_fifo_compaction_options_create() { } void rocksdb_fifo_compaction_options_set_allow_compaction( - rocksdb_fifo_compaction_options_t* fifo_opts, unsigned char allow_compaction) { + rocksdb_fifo_compaction_options_t* fifo_opts, + unsigned char allow_compaction) { fifo_opts->rep.allow_compaction = allow_compaction; } @@ -5405,6 +5558,11 @@ char* rocksdb_sst_file_metadata_get_relative_filename( return strdup(file_meta->rep->relative_filename.c_str()); } +char* rocksdb_sst_file_metadata_get_directory( + rocksdb_sst_file_metadata_t* file_meta) { + return strdup(file_meta->rep->directory.c_str()); +} + uint64_t rocksdb_sst_file_metadata_get_size( rocksdb_sst_file_metadata_t* file_meta) { return file_meta->rep->size; @@ -5623,8 +5781,7 @@ int rocksdb_transactiondb_property_int(rocksdb_transactiondb_t* db, } } -rocksdb_t* rocksdb_transactiondb_get_base_db( - rocksdb_transactiondb_t* txn_db) { +rocksdb_t* rocksdb_transactiondb_get_base_db(rocksdb_transactiondb_t* txn_db) { DB* base_db = txn_db->rep->GetBaseDB(); if (base_db != nullptr) { @@ -5636,9 +5793,7 @@ rocksdb_t* rocksdb_transactiondb_get_base_db( return nullptr; } -void rocksdb_transactiondb_close_base_db(rocksdb_t* base_db) { - delete base_db; -} +void rocksdb_transactiondb_close_base_db(rocksdb_t* base_db) { delete base_db; } rocksdb_transaction_t* rocksdb_transaction_begin( rocksdb_transactiondb_t* txn_db, @@ -6617,4 +6772,114 @@ void rocksdb_enable_manual_compaction(rocksdb_t* db) { db->rep->EnableManualCompaction(); } +rocksdb_statistics_histogram_data_t* +rocksdb_statistics_histogram_data_create() { + return new rocksdb_statistics_histogram_data_t{}; +} + +void rocksdb_statistics_histogram_data_destroy( + rocksdb_statistics_histogram_data_t* data) { + delete data; +} + +double rocksdb_statistics_histogram_data_get_median( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.median; +} + +double rocksdb_statistics_histogram_data_get_p95( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.percentile95; +} + +double rocksdb_statistics_histogram_data_get_p99( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.percentile99; +} + +double rocksdb_statistics_histogram_data_get_average( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.average; +} + +double rocksdb_statistics_histogram_data_get_std_dev( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.standard_deviation; +} + +double rocksdb_statistics_histogram_data_get_max( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.max; +} + +uint64_t rocksdb_statistics_histogram_data_get_count( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.count; +} + +uint64_t rocksdb_statistics_histogram_data_get_sum( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.sum; +} + +double rocksdb_statistics_histogram_data_get_min( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.min; +} + +void rocksdb_wait_for_compact(rocksdb_t* db, + rocksdb_wait_for_compact_options_t* options, + char** errptr) { + SaveError(errptr, db->rep->WaitForCompact(options->rep)); +} + +rocksdb_wait_for_compact_options_t* rocksdb_wait_for_compact_options_create() { + return new rocksdb_wait_for_compact_options_t; +} + +void rocksdb_wait_for_compact_options_destroy( + rocksdb_wait_for_compact_options_t* opt) { + delete opt; +} + +void rocksdb_wait_for_compact_options_set_abort_on_pause( + rocksdb_wait_for_compact_options_t* opt, unsigned char v) { + opt->rep.abort_on_pause = v; +} + +unsigned char rocksdb_wait_for_compact_options_get_abort_on_pause( + rocksdb_wait_for_compact_options_t* opt) { + return opt->rep.abort_on_pause; +} + +void rocksdb_wait_for_compact_options_set_flush( + rocksdb_wait_for_compact_options_t* opt, unsigned char v) { + opt->rep.flush = v; +} + +unsigned char rocksdb_wait_for_compact_options_get_flush( + rocksdb_wait_for_compact_options_t* opt) { + return opt->rep.flush; +} + +void rocksdb_wait_for_compact_options_set_close_db( + rocksdb_wait_for_compact_options_t* opt, unsigned char v) { + opt->rep.close_db = v; +} + +unsigned char rocksdb_wait_for_compact_options_get_close_db( + rocksdb_wait_for_compact_options_t* opt) { + return opt->rep.close_db; +} + +void rocksdb_wait_for_compact_options_set_timeout( + rocksdb_wait_for_compact_options_t* opt, uint64_t microseconds) { + opt->rep.timeout = std::chrono::microseconds(microseconds); +} + +uint64_t rocksdb_wait_for_compact_options_get_timeout( + rocksdb_wait_for_compact_options_t* opt) { + return opt->rep.timeout.count(); +} + } // end extern "C" diff --git a/db/c_test.c b/db/c_test.c index 80eb90b8a..667220496 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -3,15 +3,14 @@ found in the LICENSE file. See the AUTHORS file for names of contributors. */ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -#include +#include "rocksdb/c.h" #include #include +#include #include #include #include - -#include "rocksdb/c.h" #ifndef OS_WIN #include #endif @@ -376,6 +375,11 @@ static rocksdb_t* CheckCompaction(rocksdb_t* db, rocksdb_options_t* options, // Force compaction rocksdb_compact_range(db, NULL, 0, NULL, 0); + rocksdb_wait_for_compact_options_t* wco; + wco = rocksdb_wait_for_compact_options_create(); + rocksdb_wait_for_compact(db, wco, &err); + CheckNoError(err); + rocksdb_wait_for_compact_options_destroy(wco); // should have filtered bar, but not foo CheckGet(db, roptions, "foo", "foovalue"); CheckGet(db, roptions, "bar", NULL); @@ -709,6 +713,11 @@ int main(int argc, char** argv) { rocksdb_options_set_ratelimiter(options, rate_limiter); rocksdb_ratelimiter_destroy(rate_limiter); + rate_limiter = + rocksdb_ratelimiter_create_auto_tuned(1000 * 1024 * 1024, 100 * 1000, 10); + rocksdb_options_set_ratelimiter(options, rate_limiter); + rocksdb_ratelimiter_destroy(rate_limiter); + roptions = rocksdb_readoptions_create(); rocksdb_readoptions_set_verify_checksums(roptions, 1); rocksdb_readoptions_set_fill_cache(roptions, 1); @@ -1470,10 +1479,20 @@ int main(int argc, char** argv) { CheckCondition(cflen == 2); rocksdb_list_column_families_destroy(column_fams, cflen); - rocksdb_options_t* cf_options = rocksdb_options_create(); + rocksdb_options_t* cf_options_1 = rocksdb_options_create(); + rocksdb_options_t* cf_options_2 = rocksdb_options_create(); + + // use dbpathname2 as the cf_path for "cf1" + rocksdb_dbpath_t* dbpath2; + char dbpathname2[200]; + snprintf(dbpathname2, sizeof(dbpathname2), "%s/rocksdb_c_test-%d-dbpath2", + GetTempDir(), ((int)geteuid())); + dbpath2 = rocksdb_dbpath_create(dbpathname2, 1024 * 1024); + const rocksdb_dbpath_t* cf_paths[1] = {dbpath2}; + rocksdb_options_set_cf_paths(cf_options_2, cf_paths, 1); const char* cf_names[2] = {"default", "cf1"}; - const rocksdb_options_t* cf_opts[2] = {cf_options, cf_options}; + const rocksdb_options_t* cf_opts[2] = {cf_options_1, cf_options_2}; rocksdb_column_family_handle_t* handles[2]; LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names, @@ -1501,6 +1520,37 @@ int main(int argc, char** argv) { rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create(); rocksdb_flushoptions_set_wait(flush_options, 1); rocksdb_flush_cf(db, flush_options, handles[1], &err); + + // make sure all files in "cf1" are under the specified cf path + { + rocksdb_column_family_metadata_t* cf_meta = + rocksdb_get_column_family_metadata_cf(db, handles[1]); + size_t cf_file_count = rocksdb_column_family_metadata_get_size(cf_meta); + assert(cf_file_count > 0); + size_t level_count = + rocksdb_column_family_metadata_get_level_count(cf_meta); + assert(level_count > 0); + for (size_t l = 0; l < level_count; ++l) { + rocksdb_level_metadata_t* level_meta = + rocksdb_column_family_metadata_get_level_metadata(cf_meta, l); + assert(level_meta); + + size_t file_count = rocksdb_level_metadata_get_file_count(level_meta); + for (size_t f = 0; f < file_count; ++f) { + rocksdb_sst_file_metadata_t* file_meta = + rocksdb_level_metadata_get_sst_file_metadata(level_meta, f); + assert(file_meta); + char* file_path = rocksdb_sst_file_metadata_get_directory(file_meta); + assert(strcmp(file_path, dbpathname2) == 0); + Free(&file_path); + rocksdb_sst_file_metadata_destroy(file_meta); + } + rocksdb_level_metadata_destroy(level_meta); + } + + rocksdb_column_family_metadata_destroy(cf_meta); + } + CheckNoError(err) rocksdb_flushoptions_destroy(flush_options); CheckGetCF(db, roptions, handles[1], "foo", "hello"); @@ -1664,7 +1714,9 @@ int main(int argc, char** argv) { } rocksdb_destroy_db(options, dbname, &err); rocksdb_options_destroy(db_options); - rocksdb_options_destroy(cf_options); + rocksdb_options_destroy(cf_options_1); + rocksdb_options_destroy(cf_options_2); + rocksdb_dbpath_destroy(dbpath2); } StartPhase("prefix"); @@ -1869,6 +1921,10 @@ int main(int argc, char** argv) { CheckCondition(2.0 == rocksdb_options_get_max_bytes_for_level_multiplier(o)); + rocksdb_options_set_periodic_compaction_seconds(o, 100000); + CheckCondition(100000 == + rocksdb_options_get_periodic_compaction_seconds(o)); + rocksdb_options_set_skip_stats_update_on_db_open(o, 1); CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o)); @@ -2061,6 +2117,15 @@ int main(int argc, char** argv) { CheckCondition(29.0 == rocksdb_options_get_experimental_mempurge_threshold(o)); + CheckCondition(rocksdb_statistics_level_disable_all == + rocksdb_options_get_statistics_level(o)); + rocksdb_options_enable_statistics(o); + CheckCondition(rocksdb_statistics_level_disable_all != + rocksdb_options_get_statistics_level(o)); + rocksdb_options_set_statistics_level(o, rocksdb_statistics_level_all); + CheckCondition(rocksdb_statistics_level_all == + rocksdb_options_get_statistics_level(o)); + /* Blob Options */ rocksdb_options_set_enable_blob_files(o, 1); CheckCondition(1 == rocksdb_options_get_enable_blob_files(o)); @@ -2290,6 +2355,12 @@ int main(int argc, char** argv) { CheckCondition(2.0 == rocksdb_options_get_max_bytes_for_level_multiplier(o)); + rocksdb_options_set_periodic_compaction_seconds(copy, 8000); + CheckCondition(8000 == + rocksdb_options_get_periodic_compaction_seconds(copy)); + CheckCondition(100000 == + rocksdb_options_get_periodic_compaction_seconds(o)); + rocksdb_options_set_skip_stats_update_on_db_open(copy, 0); CheckCondition(0 == rocksdb_options_get_skip_stats_update_on_db_open(copy)); CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o)); @@ -3122,12 +3193,12 @@ int main(int argc, char** argv) { CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", NULL); CheckTxnDBPinGetCF(txn_db, roptions, cfh, "cf_foo", NULL); - - //memory usage + // memory usage rocksdb_t* base_db = rocksdb_transactiondb_get_base_db(txn_db); rocksdb_memory_consumers_t* consumers = rocksdb_memory_consumers_create(); rocksdb_memory_consumers_add_db(consumers, base_db); - rocksdb_memory_usage_t* usage = rocksdb_approximate_memory_usage_create(consumers, &err); + rocksdb_memory_usage_t* usage = + rocksdb_approximate_memory_usage_create(consumers, &err); CheckNoError(err); rocksdb_approximate_memory_usage_destroy(usage); rocksdb_memory_consumers_destroy(consumers); @@ -3614,6 +3685,126 @@ int main(int argc, char** argv) { rocksdb_readoptions_destroy(ropts); } + StartPhase("statistics"); + { + const uint32_t BYTES_WRITTEN_TICKER = 40; + const uint32_t DB_WRITE_HIST = 1; + + rocksdb_statistics_histogram_data_t* hist = + rocksdb_statistics_histogram_data_create(); + { + // zero by default + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_median(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p95(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p99(hist)); + CheckCondition(0.0 == + rocksdb_statistics_histogram_data_get_average(hist)); + CheckCondition(0.0 == + rocksdb_statistics_histogram_data_get_std_dev(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_max(hist)); + CheckCondition(0 == rocksdb_statistics_histogram_data_get_count(hist)); + CheckCondition(0 == rocksdb_statistics_histogram_data_get_sum(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_min(hist)); + } + + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + rocksdb_options_enable_statistics(options); + rocksdb_options_set_statistics_level(options, rocksdb_statistics_level_all); + + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + + CheckCondition(0 == rocksdb_options_statistics_get_ticker_count( + options, BYTES_WRITTEN_TICKER)); + rocksdb_options_statistics_get_histogram_data(options, DB_WRITE_HIST, hist); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_median(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p95(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p99(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_average(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_std_dev(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_max(hist)); + CheckCondition(0 == rocksdb_statistics_histogram_data_get_count(hist)); + CheckCondition(0 == rocksdb_statistics_histogram_data_get_sum(hist)); + + int i; + for (i = 0; i < 10; ++i) { + char key = '0' + (char)i; + rocksdb_put(db, woptions, &key, 1, "", 1, &err); + CheckNoError(err); + } + CheckCondition(0 != rocksdb_options_statistics_get_ticker_count( + options, BYTES_WRITTEN_TICKER)); + rocksdb_options_statistics_get_histogram_data(options, DB_WRITE_HIST, hist); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_median(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_p95(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_p99(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_average(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_std_dev(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_max(hist)); + CheckCondition(0 != rocksdb_statistics_histogram_data_get_count(hist)); + CheckCondition(0 != rocksdb_statistics_histogram_data_get_sum(hist)); + + rocksdb_statistics_histogram_data_destroy(hist); + } + + StartPhase("wait_for_compact_options"); + { + rocksdb_wait_for_compact_options_t* wco; + wco = rocksdb_wait_for_compact_options_create(); + + rocksdb_wait_for_compact_options_set_abort_on_pause(wco, 1); + CheckCondition(1 == + rocksdb_wait_for_compact_options_get_abort_on_pause(wco)); + + rocksdb_wait_for_compact_options_set_flush(wco, 1); + CheckCondition(1 == rocksdb_wait_for_compact_options_get_flush(wco)); + + rocksdb_wait_for_compact_options_set_close_db(wco, 1); + CheckCondition(1 == rocksdb_wait_for_compact_options_get_close_db(wco)); + + rocksdb_wait_for_compact_options_set_timeout(wco, 342); + CheckCondition(342 == rocksdb_wait_for_compact_options_get_timeout(wco)); + + rocksdb_wait_for_compact_options_destroy(wco); + } + StartPhase("wait_for_compact"); + { + rocksdb_wait_for_compact_options_t* wco; + wco = rocksdb_wait_for_compact_options_create(); + rocksdb_wait_for_compact_options_set_flush(wco, 1); + + rocksdb_wait_for_compact(db, wco, &err); + CheckNoError(err); + rocksdb_wait_for_compact_options_destroy(wco); + } + + StartPhase("write_buffer_manager"); + { + rocksdb_cache_t* lru; + lru = rocksdb_cache_create_lru(100); + + rocksdb_write_buffer_manager_t* write_buffer_manager; + write_buffer_manager = + rocksdb_write_buffer_manager_create_with_cache(200, lru, false); + + CheckCondition(true == + rocksdb_write_buffer_manager_enabled(write_buffer_manager)); + CheckCondition(true == rocksdb_write_buffer_manager_cost_to_cache( + write_buffer_manager)); + CheckCondition( + 200 == rocksdb_write_buffer_manager_buffer_size(write_buffer_manager)); + + rocksdb_write_buffer_manager_set_buffer_size(write_buffer_manager, 300); + CheckCondition( + 300 == rocksdb_write_buffer_manager_buffer_size(write_buffer_manager)); + + rocksdb_write_buffer_manager_destroy(write_buffer_manager); + rocksdb_cache_destroy(lru); + } + StartPhase("cancel_all_background_work"); rocksdb_cancel_all_background_work(db, 1); diff --git a/db/column_family.cc b/db/column_family.cc index 23349ba58..2be522341 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -383,7 +383,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, const uint64_t kAdjustedTtl = 30 * 24 * 60 * 60; if (result.ttl == kDefaultTtl) { if (is_block_based_table) { - // For FIFO, max_open_files is checked in ValidateOptions(). + // FIFO also requires max_open_files=-1, which is checked in + // ValidateOptions(). result.ttl = kAdjustedTtl; } else { result.ttl = 0; @@ -391,20 +392,20 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, } const uint64_t kAdjustedPeriodicCompSecs = 30 * 24 * 60 * 60; - - // Turn on periodic compactions and set them to occur once every 30 days if - // compaction filters are used and periodic_compaction_seconds is set to the - // default value. - if (result.compaction_style != kCompactionStyleFIFO) { + if (result.compaction_style == kCompactionStyleLevel) { if ((result.compaction_filter != nullptr || result.compaction_filter_factory != nullptr) && result.periodic_compaction_seconds == kDefaultPeriodicCompSecs && is_block_based_table) { result.periodic_compaction_seconds = kAdjustedPeriodicCompSecs; } - } else { - if (result.periodic_compaction_seconds != kDefaultPeriodicCompSecs && - result.periodic_compaction_seconds > 0) { + } else if (result.compaction_style == kCompactionStyleUniversal) { + if (result.periodic_compaction_seconds == kDefaultPeriodicCompSecs && + is_block_based_table) { + result.periodic_compaction_seconds = kAdjustedPeriodicCompSecs; + } + } else if (result.compaction_style == kCompactionStyleFIFO) { + if (result.periodic_compaction_seconds != kDefaultPeriodicCompSecs) { ROCKS_LOG_WARN( db_options.info_log.get(), "periodic_compaction_seconds does not support FIFO compaction. You" @@ -412,15 +413,14 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, } } - // TTL compactions would work similar to Periodic Compactions in Universal in - // most of the cases. So, if ttl is set, execute the periodic compaction - // codepath. - if (result.compaction_style == kCompactionStyleUniversal && result.ttl != 0) { - if (result.periodic_compaction_seconds != 0) { + // For universal compaction, `ttl` and `periodic_compaction_seconds` mean the + // same thing, take the stricter value. + if (result.compaction_style == kCompactionStyleUniversal) { + if (result.periodic_compaction_seconds == 0) { + result.periodic_compaction_seconds = result.ttl; + } else if (result.ttl != 0) { result.periodic_compaction_seconds = std::min(result.ttl, result.periodic_compaction_seconds); - } else { - result.periodic_compaction_seconds = result.ttl; } } @@ -476,6 +476,7 @@ void SuperVersion::Init(ColumnFamilyData* new_cfd, MemTable* new_mem, mem = new_mem; imm = new_imm; current = new_current; + full_history_ts_low = cfd->GetFullHistoryTsLow(); cfd->Ref(); mem->Ref(); imm->Ref(); @@ -832,8 +833,8 @@ std::unique_ptr SetupDelay( return write_controller->GetDelayToken(write_rate); } -int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger, - int level0_slowdown_writes_trigger) { +int GetL0FileCountForCompactionSpeedup(int level0_file_num_compaction_trigger, + int level0_slowdown_writes_trigger) { // SanitizeOptions() ensures it. assert(level0_file_num_compaction_trigger <= level0_slowdown_writes_trigger); @@ -863,6 +864,43 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger, return static_cast(res); } } + +uint64_t GetPendingCompactionBytesForCompactionSpeedup( + const MutableCFOptions& mutable_cf_options, + const VersionStorageInfo* vstorage) { + // Compaction debt relatively large compared to the stable (bottommost) data + // size indicates compaction fell behind. + const uint64_t kBottommostSizeDivisor = 8; + // Meaningful progress toward the slowdown trigger is another good indication. + const uint64_t kSlowdownTriggerDivisor = 4; + + uint64_t bottommost_files_size = 0; + for (const auto& level_and_file : vstorage->BottommostFiles()) { + bottommost_files_size += level_and_file.second->fd.GetFileSize(); + } + + // Slowdown trigger might be zero but that means compaction speedup should + // always happen (undocumented/historical), so no special treatment is needed. + uint64_t slowdown_threshold = + mutable_cf_options.soft_pending_compaction_bytes_limit / + kSlowdownTriggerDivisor; + + // Size of zero, however, should not be used to decide to speedup compaction. + if (bottommost_files_size == 0) { + return slowdown_threshold; + } + + uint64_t size_threshold = bottommost_files_size / kBottommostSizeDivisor; + return std::min(size_threshold, slowdown_threshold); +} + +uint64_t GetMarkedFileCountForCompactionSpeedup() { + // When just one file is marked, it is not clear that parallel compaction will + // help the compaction that the user nicely requested to happen sooner. When + // multiple files are marked, however, it is pretty clearly helpful, except + // for the rare case in which a single compaction grabs all the marked files. + return 2; +} } // anonymous namespace std::pair @@ -1018,7 +1056,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions( } else { assert(write_stall_condition == WriteStallCondition::kNormal); if (vstorage->l0_delay_trigger_count() >= - GetL0ThresholdSpeedupCompaction( + GetL0FileCountForCompactionSpeedup( mutable_cf_options.level0_file_num_compaction_trigger, mutable_cf_options.level0_slowdown_writes_trigger)) { write_controller_token_ = @@ -1028,22 +1066,32 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions( "[%s] Increasing compaction threads because we have %d level-0 " "files ", name_.c_str(), vstorage->l0_delay_trigger_count()); - } else if (vstorage->estimated_compaction_needed_bytes() >= - mutable_cf_options.soft_pending_compaction_bytes_limit / 4) { - // Increase compaction threads if bytes needed for compaction exceeds - // 1/4 of threshold for slowing down. + } else if (mutable_cf_options.soft_pending_compaction_bytes_limit == 0) { // If soft pending compaction byte limit is not set, always speed up // compaction. write_controller_token_ = write_controller->GetCompactionPressureToken(); - if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) { - ROCKS_LOG_INFO( - ioptions_.logger, - "[%s] Increasing compaction threads because of estimated pending " - "compaction " - "bytes %" PRIu64, - name_.c_str(), vstorage->estimated_compaction_needed_bytes()); - } + } else if (vstorage->estimated_compaction_needed_bytes() >= + GetPendingCompactionBytesForCompactionSpeedup( + mutable_cf_options, vstorage)) { + write_controller_token_ = + write_controller->GetCompactionPressureToken(); + ROCKS_LOG_INFO( + ioptions_.logger, + "[%s] Increasing compaction threads because of estimated pending " + "compaction " + "bytes %" PRIu64, + name_.c_str(), vstorage->estimated_compaction_needed_bytes()); + } else if (uint64_t(vstorage->FilesMarkedForCompaction().size()) >= + GetMarkedFileCountForCompactionSpeedup()) { + write_controller_token_ = + write_controller->GetCompactionPressureToken(); + ROCKS_LOG_INFO( + ioptions_.logger, + "[%s] Increasing compaction threads because we have %" PRIu64 + " files marked for compaction", + name_.c_str(), + uint64_t(vstorage->FilesMarkedForCompaction().size())); } else { write_controller_token_.reset(); } @@ -1118,7 +1166,7 @@ Compaction* ColumnFamilyData::PickCompaction( GetName(), mutable_options, mutable_db_options, current_->storage_info(), log_buffer); if (result != nullptr) { - result->SetInputVersion(current_); + result->FinalizeInputInfo(current_); } return result; } @@ -1202,7 +1250,7 @@ Compaction* ColumnFamilyData::CompactRange( compact_range_options, begin, end, compaction_end, conflict, max_file_num_to_ignore, trim_ts); if (result != nullptr) { - result->SetInputVersion(current_); + result->FinalizeInputInfo(current_); } TEST_SYNC_POINT("ColumnFamilyData::CompactRange:Return"); return result; @@ -1283,8 +1331,6 @@ void ColumnFamilyData::InstallSuperVersion( new_superversion->Init(this, mem_, imm_.current(), current_); SuperVersion* old_superversion = super_version_; super_version_ = new_superversion; - ++super_version_number_; - super_version_->version_number = super_version_number_; if (old_superversion == nullptr || old_superversion->current != current() || old_superversion->mem != mem_ || old_superversion->imm != imm_.current()) { @@ -1319,6 +1365,8 @@ void ColumnFamilyData::InstallSuperVersion( sv_context->superversions_to_free.push_back(old_superversion); } } + ++super_version_number_; + super_version_->version_number = super_version_number_; } void ColumnFamilyData::ResetThreadLocalSuperVersions() { @@ -1376,6 +1424,33 @@ Status ColumnFamilyData::ValidateOptions( } } + const auto* ucmp = cf_options.comparator; + assert(ucmp); + if (ucmp->timestamp_size() > 0 && + !cf_options.persist_user_defined_timestamps) { + if (db_options.atomic_flush) { + return Status::NotSupported( + "Not persisting user-defined timestamps feature is not supported" + "in combination with atomic flush."); + } + if (db_options.allow_concurrent_memtable_write) { + return Status::NotSupported( + "Not persisting user-defined timestamps feature is not supported" + " in combination with concurrent memtable write."); + } + const char* comparator_name = cf_options.comparator->Name(); + size_t name_size = strlen(comparator_name); + const char* suffix = ".u64ts"; + size_t suffix_size = strlen(suffix); + if (name_size <= suffix_size || + strcmp(comparator_name + name_size - suffix_size, suffix) != 0) { + return Status::NotSupported( + "Not persisting user-defined timestamps" + "feature only support user-defined timestamps formatted as " + "uint64_t."); + } + } + if (cf_options.enable_blob_garbage_collection) { if (cf_options.blob_garbage_collection_age_cutoff < 0.0 || cf_options.blob_garbage_collection_age_cutoff > 1.0) { @@ -1515,6 +1590,34 @@ FSDirectory* ColumnFamilyData::GetDataDir(size_t path_id) const { return data_dirs_[path_id].get(); } +bool ColumnFamilyData::ShouldPostponeFlushToRetainUDT( + uint64_t max_memtable_id) { + const Comparator* ucmp = user_comparator(); + const size_t ts_sz = ucmp->timestamp_size(); + if (ts_sz == 0 || ioptions_.persist_user_defined_timestamps) { + return false; + } + // If users set the `persist_user_defined_timestamps` flag to false, they + // should also set the `full_history_ts_low` flag to indicate the range of + // user-defined timestamps to retain in memory. Otherwise, we do not + // explicitly postpone flush to retain UDTs. + const std::string& full_history_ts_low = GetFullHistoryTsLow(); + if (full_history_ts_low.empty()) { + return false; + } + for (const Slice& table_newest_udt : + imm()->GetTablesNewestUDT(max_memtable_id)) { + assert(table_newest_udt.size() == full_history_ts_low.size()); + // Checking the newest UDT contained in MemTable with ascending ID up to + // `max_memtable_id`. Return immediately on finding the first MemTable that + // needs postponing. + if (ucmp->CompareTimestamp(table_newest_udt, full_history_ts_low) >= 0) { + return true; + } + } + return false; +} + void ColumnFamilyData::RecoverEpochNumbers() { assert(current_); auto* vstorage = current_->storage_info(); @@ -1692,4 +1795,20 @@ const Comparator* GetColumnFamilyUserComparator( return nullptr; } +const ImmutableOptions& GetImmutableOptions(ColumnFamilyHandle* column_family) { + assert(column_family); + + ColumnFamilyHandleImpl* const handle = + static_cast_with_check(column_family); + assert(handle); + + const ColumnFamilyData* const cfd = handle->cfd(); + assert(cfd); + + const ImmutableOptions* ioptions = cfd->ioptions(); + assert(ioptions); + + return *ioptions; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/column_family.h b/db/column_family.h index 05f126ae6..2a38feb73 100644 --- a/db/column_family.h +++ b/db/column_family.h @@ -211,6 +211,12 @@ struct SuperVersion { // Version number of the current SuperVersion uint64_t version_number; WriteStallCondition write_stall_condition; + // Each time `full_history_ts_low` collapses history, a new SuperVersion is + // installed. This field tracks the effective `full_history_ts_low` for that + // SuperVersion, to be used by read APIs for sanity checks. This field is + // immutable once SuperVersion is installed. For column family that doesn't + // enable UDT feature, this is an empty string. + std::string full_history_ts_low; // should be called outside the mutex SuperVersion() = default; @@ -506,6 +512,12 @@ class ColumnFamilyData { return full_history_ts_low_; } + // REQUIRES: DB mutex held. + // Return true if flushing up to MemTables with ID `max_memtable_id` + // should be postponed to retain user-defined timestamps according to the + // user's setting. Called by background flush job. + bool ShouldPostponeFlushToRetainUDT(uint64_t max_memtable_id); + ThreadLocalPtr* TEST_GetLocalSV() { return local_sv_.get(); } WriteBufferManager* write_buffer_mgr() { return write_buffer_manager_; } std::shared_ptr @@ -513,8 +525,6 @@ class ColumnFamilyData { return file_metadata_cache_res_mgr_; } - SequenceNumber GetFirstMemtableSequenceNumber() const; - static const uint32_t kDummyColumnFamilyDataId; // Keep track of whether the mempurge feature was ever used. @@ -867,4 +877,7 @@ extern uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family); extern const Comparator* GetColumnFamilyUserComparator( ColumnFamilyHandle* column_family); +extern const ImmutableOptions& GetImmutableOptions( + ColumnFamilyHandle* column_family); + } // namespace ROCKSDB_NAMESPACE diff --git a/db/column_family_test.cc b/db/column_family_test.cc index 08393c350..cf7327287 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -8,6 +8,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include +#include #include #include #include @@ -17,6 +18,7 @@ #include "options/options_parser.h" #include "port/port.h" #include "port/stack_trace.h" +#include "rocksdb/comparator.h" #include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/env.h" @@ -26,6 +28,7 @@ #include "test_util/testharness.h" #include "test_util/testutil.h" #include "util/coding.h" +#include "util/defer.h" #include "util/string_util.h" #include "utilities/fault_injection_env.h" #include "utilities/merge_operators.h" @@ -63,6 +66,9 @@ class ColumnFamilyTestBase : public testing::Test { db_options_.create_if_missing = true; db_options_.fail_if_options_file_error = true; db_options_.env = env_; + } + + void SetUp() override { EXPECT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_))); } @@ -2165,13 +2171,57 @@ TEST_P(ColumnFamilyTest, FlushStaleColumnFamilies) { Close(); } +namespace { +struct CountOptionsFilesFs : public FileSystemWrapper { + explicit CountOptionsFilesFs(const std::shared_ptr& t) + : FileSystemWrapper(t) {} + const char* Name() const override { return "CountOptionsFilesFs"; } + + IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override { + if (f.find("OPTIONS-") != std::string::npos) { + options_files_created.fetch_add(1, std::memory_order_relaxed); + } + return FileSystemWrapper::NewWritableFile(f, file_opts, r, dbg); + } + + std::atomic options_files_created{}; +}; +} // namespace + TEST_P(ColumnFamilyTest, CreateMissingColumnFamilies) { - Status s = TryOpen({"one", "two"}); - ASSERT_TRUE(!s.ok()); + // Can't accidentally add CFs to an existing DB + Open(); + Close(); + ASSERT_FALSE(db_options_.create_missing_column_families); + ASSERT_NOK(TryOpen({"one", "two"})); + + // Nor accidentally create in a new DB + Destroy(); + db_options_.create_if_missing = true; + ASSERT_NOK(TryOpen({"one", "two"})); + + // Only with the option (new DB case) db_options_.create_missing_column_families = true; - s = TryOpen({"default", "one", "two"}); - ASSERT_TRUE(s.ok()); + // Also setup to count number of options files created (see check below) + auto my_fs = + std::make_shared(db_options_.env->GetFileSystem()); + auto my_env = std::make_unique(db_options_.env, my_fs); + SaveAndRestore save_restore_env(&db_options_.env, my_env.get()); + + ASSERT_OK(TryOpen({"default", "one", "two"})); + Close(); + + // An older version would write an updated options file for each column + // family created under create_missing_column_families, which would be + // quadratic I/O in the number of column families. + ASSERT_EQ(my_fs->options_files_created.load(), 1); + + // Add to existing DB case + ASSERT_OK(TryOpen({"default", "one", "two", "three", "four"})); Close(); + ASSERT_EQ(my_fs->options_files_created.load(), 2); } TEST_P(ColumnFamilyTest, SanitizeOptions) { @@ -2423,7 +2473,10 @@ void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id, } } // anonymous namespace -TEST_P(ColumnFamilyTest, CreateAndDropRace) { +// This test attempts to set up a race condition in a way that is no longer +// possible, causing the test to hang. If DBImpl::options_mutex_ is removed +// in the future, this test might become relevant again. +TEST_P(ColumnFamilyTest, DISABLED_CreateAndDropRace) { const int kCfCount = 5; std::vector cf_opts; std::vector comparators; @@ -2485,6 +2538,53 @@ TEST_P(ColumnFamilyTest, CreateAndDropRace) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); } +TEST_P(ColumnFamilyTest, CreateAndDropPeriodicRace) { + // This is a mini-stress test looking for inconsistency between the set of + // CFs in the DB, particularly whether any use preserve_internal_time_seconds, + // and whether that is accurately reflected in the periodic task setup. + constexpr size_t kNumThreads = 12; + std::vector threads; + bool last_cf_on = Random::GetTLSInstance()->OneIn(2); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RegisterRecordSeqnoTimeWorker:BeforePeriodicTaskType", + [&](void* /*arg*/) { std::this_thread::yield(); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_EQ(column_family_options_.preserve_internal_time_seconds, 0U); + ColumnFamilyOptions other_opts = column_family_options_; + ColumnFamilyOptions last_opts = column_family_options_; + (last_cf_on ? last_opts : other_opts).preserve_internal_time_seconds = + 1000000; + Open(); + + for (size_t i = 0; i < kNumThreads; i++) { + threads.emplace_back([this, &other_opts, i]() { + ColumnFamilyHandle* cfh; + ASSERT_OK(db_->CreateColumnFamily(other_opts, std::to_string(i), &cfh)); + ASSERT_OK(db_->DropColumnFamily(cfh)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh)); + }); + } + + ColumnFamilyHandle* last_cfh; + ASSERT_OK(db_->CreateColumnFamily(last_opts, "last", &last_cfh)); + + for (auto& t : threads) { + t.join(); + } + + bool task_enabled = dbfull()->TEST_GetPeriodicTaskScheduler().TEST_HasTask( + PeriodicTaskType::kRecordSeqnoTime); + ASSERT_EQ(last_cf_on, task_enabled); + + ASSERT_OK(db_->DropColumnFamily(last_cfh)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(last_cfh)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) { const uint64_t kBaseRate = 800000u; db_options_.delayed_write_rate = kBaseRate; @@ -2871,6 +2971,112 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) { ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); } +TEST_P(ColumnFamilyTest, CompactionSpeedupForCompactionDebt) { + db_options_.max_background_compactions = 6; + Open(); + ColumnFamilyData* cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + MutableCFOptions mutable_cf_options(column_family_options_); + mutable_cf_options.soft_pending_compaction_bytes_limit = + std::numeric_limits::max(); + + { + // No bottommost data, so debt ratio cannot trigger speedup. + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + vstorage->TEST_set_estimated_compaction_needed_bytes(1048576 /* 1MB */); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + } + + // Add a tiny amount of bottommost data. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "bar")); + ASSERT_OK(db_->Flush(FlushOptions())); + + { + // 1MB debt is way bigger than bottommost data so definitely triggers + // speedup. + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + vstorage->TEST_set_estimated_compaction_needed_bytes(1048576 /* 1MB */); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + // Eight bytes is way smaller than bottommost data so definitely does not + // trigger speedup. + vstorage->TEST_set_estimated_compaction_needed_bytes(8); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + } +} + +TEST_P(ColumnFamilyTest, CompactionSpeedupForMarkedFiles) { + const int kParallelismLimit = 3; + class AlwaysCompactTpc : public TablePropertiesCollector { + public: + Status Finish(UserCollectedProperties* /* properties */) override { + return Status::OK(); + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } + + const char* Name() const override { return "AlwaysCompactTpc"; } + + bool NeedCompact() const override { return true; } + }; + + class AlwaysCompactTpcf : public TablePropertiesCollectorFactory { + public: + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /* context */) override { + return new AlwaysCompactTpc(); + } + + const char* Name() const override { return "AlwaysCompactTpcf"; } + }; + + column_family_options_.num_levels = 2; + column_family_options_.table_properties_collector_factories.emplace_back( + std::make_shared()); + db_options_.max_background_compactions = kParallelismLimit; + Open(); + + // Make a nonempty last level. Only marked files in upper levels count. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "bar")); + ASSERT_OK(db_->Flush(FlushOptions())); + WaitForCompaction(); + AssertFilesPerLevel("0,1", 0 /* cf */); + + // Block the compaction thread pool so marked files accumulate in L0. + test::SleepingBackgroundTask sleeping_tasks[kParallelismLimit]; + for (int i = 0; i < kParallelismLimit; i++) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_tasks[i], Env::Priority::LOW); + sleeping_tasks[i].WaitUntilSleeping(); + } + + // Zero marked upper-level files. No speedup. + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + AssertFilesPerLevel("0,1", 0 /* cf */); + + // One marked upper-level file. No speedup. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "bar")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + AssertFilesPerLevel("1,1", 0 /* cf */); + + // Two marked upper-level files. Speedup. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "bar")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(kParallelismLimit, dbfull()->TEST_BGCompactionsAllowed()); + AssertFilesPerLevel("2,1", 0 /* cf */); + + for (int i = 0; i < kParallelismLimit; i++) { + sleeping_tasks[i].WakeUp(); + sleeping_tasks[i].WaitUntilDone(); + } +} + TEST_P(ColumnFamilyTest, CreateAndDestroyOptions) { std::unique_ptr cfo(new ColumnFamilyOptions()); ColumnFamilyHandle* cfh; @@ -3380,6 +3586,205 @@ TEST(ColumnFamilyTest, ValidateMemtableKVChecksumOption) { ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); } +// Tests the flushing behavior of a column family to retain user-defined +// timestamp when `persist_user_defined_timestamp` is false. +class ColumnFamilyRetainUDTTest : public ColumnFamilyTestBase { + public: + ColumnFamilyRetainUDTTest() : ColumnFamilyTestBase(kLatestFormatVersion) {} + + void SetUp() override { + db_options_.allow_concurrent_memtable_write = false; + column_family_options_.comparator = + test::BytewiseComparatorWithU64TsWrapper(); + column_family_options_.persist_user_defined_timestamps = false; + ColumnFamilyTestBase::SetUp(); + } + + Status Put(int cf, const std::string& key, const std::string& ts, + const std::string& value) { + return db_->Put(WriteOptions(), handles_[cf], Slice(key), Slice(ts), + Slice(value)); + } +}; + +class TestTsComparator : public Comparator { + public: + TestTsComparator() : Comparator(8 /*ts_sz*/) {} + + int Compare(const ROCKSDB_NAMESPACE::Slice& /*a*/, + const ROCKSDB_NAMESPACE::Slice& /*b*/) const override { + return 0; + } + const char* Name() const override { return "TestTs"; } + void FindShortestSeparator( + std::string* /*start*/, + const ROCKSDB_NAMESPACE::Slice& /*limit*/) const override {} + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +TEST_F(ColumnFamilyRetainUDTTest, SanityCheck) { + Open(); + ColumnFamilyOptions cf_options; + cf_options.persist_user_defined_timestamps = false; + TestTsComparator test_comparator; + cf_options.comparator = &test_comparator; + ColumnFamilyHandle* handle; + // Not persisting user-defined timestamps feature only supports user-defined + // timestamps formatted as uint64_t. + ASSERT_TRUE( + db_->CreateColumnFamily(cf_options, "pikachu", &handle).IsNotSupported()); + + Destroy(); + // Not persisting user-defined timestamps feature doesn't work in combination + // with atomic flush. + db_options_.atomic_flush = true; + ASSERT_TRUE(TryOpen({"default"}).IsNotSupported()); + + // Not persisting user-defined timestamps feature doesn't work in combination + // with concurrent memtable write. + db_options_.atomic_flush = false; + db_options_.allow_concurrent_memtable_write = true; + ASSERT_TRUE(TryOpen({"default"}).IsNotSupported()); + Close(); +} + +TEST_F(ColumnFamilyRetainUDTTest, FullHistoryTsLowNotSet) { + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto reschedule_count = *static_cast(arg); + ASSERT_EQ(1, reschedule_count); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Open(); + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(Put(0, "foo", write_ts, "v1")); + // No `full_history_ts_low` explicitly set by user, flush is continued + // without checking if its UDTs expired. + ASSERT_OK(Flush(0)); + + // After flush, `full_history_ts_low` should be automatically advanced to + // the effective cutoff timestamp: write_ts + 1 + std::string cutoff_ts; + PutFixed64(&cutoff_ts, 2); + std::string effective_full_history_ts_low; + ASSERT_OK( + db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low)); + ASSERT_EQ(cutoff_ts, effective_full_history_ts_low); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(ColumnFamilyRetainUDTTest, AllKeysExpired) { + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto reschedule_count = *static_cast(arg); + ASSERT_EQ(1, reschedule_count); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Open(); + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(Put(0, "foo", write_ts, "v1")); + std::string cutoff_ts; + PutFixed64(&cutoff_ts, 3); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts)); + // All keys expired w.r.t the configured `full_history_ts_low`, flush continue + // without the need for a re-schedule. + ASSERT_OK(Flush(0)); + + // `full_history_ts_low` stays unchanged after flush. + std::string effective_full_history_ts_low; + ASSERT_OK( + db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low)); + ASSERT_EQ(cutoff_ts, effective_full_history_ts_low); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} +TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushToAvoidWriteStall) { + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto reschedule_count = *static_cast(arg); + ASSERT_EQ(1, reschedule_count); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Open(); + std::string cutoff_ts; + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(Put(0, "foo", write_ts, "v1")); + PutFixed64(&cutoff_ts, 1); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts)); + ASSERT_OK(db_->SetOptions(handles_[0], {{"max_write_buffer_number", "1"}})); + // Not all keys expired, but flush is continued without a re-schedule because + // of risk of write stall. + ASSERT_OK(Flush(0)); + + // After flush, `full_history_ts_low` should be automatically advanced to + // the effective cutoff timestamp: write_ts + 1 + std::string effective_full_history_ts_low; + ASSERT_OK( + db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low)); + + cutoff_ts.clear(); + PutFixed64(&cutoff_ts, 2); + ASSERT_EQ(cutoff_ts, effective_full_history_ts_low); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushRescheduled) { + std::string cutoff_ts; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::AfterRetainUDTReschedule:cb", [&](void* /*arg*/) { + // Increasing full_history_ts_low so all keys expired after the initial + // FlushRequest is rescheduled + cutoff_ts.clear(); + PutFixed64(&cutoff_ts, 3); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts)); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto reschedule_count = *static_cast(arg); + ASSERT_EQ(2, reschedule_count); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Open(); + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(Put(0, "foo", write_ts, "v1")); + PutFixed64(&cutoff_ts, 1); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts)); + // Not all keys expired, and there is no risk of write stall. Flush is + // rescheduled. The actual flush happens after `full_history_ts_low` is + // increased to mark all keys expired. + ASSERT_OK(Flush(0)); + + std::string effective_full_history_ts_low; + ASSERT_OK( + db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low)); + // `full_history_ts_low` stays unchanged. + ASSERT_EQ(cutoff_ts, effective_full_history_ts_low); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/compact_files_test.cc b/db/compact_files_test.cc index 5ca80d9aa..2d53f2b99 100644 --- a/db/compact_files_test.cc +++ b/db/compact_files_test.cc @@ -345,7 +345,7 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) { return true; } std::string res; - db_->Get(ReadOptions(), "", &res); + EXPECT_TRUE(db_->Get(ReadOptions(), "", &res).IsNotFound()); return true; } diff --git a/db/compaction/compaction.cc b/db/compaction/compaction.cc index ceed9d104..bbab8f79f 100644 --- a/db/compaction/compaction.cc +++ b/db/compaction/compaction.cc @@ -13,6 +13,7 @@ #include #include "db/column_family.h" +#include "logging/logging.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/sst_partitioner.h" #include "test_util/sync_point.h" @@ -114,6 +115,42 @@ void Compaction::GetBoundaryKeys( } } +void Compaction::GetBoundaryInternalKeys( + VersionStorageInfo* vstorage, + const std::vector& inputs, InternalKey* smallest_key, + InternalKey* largest_key, int exclude_level) { + bool initialized = false; + const InternalKeyComparator* icmp = vstorage->InternalComparator(); + for (size_t i = 0; i < inputs.size(); ++i) { + if (inputs[i].files.empty() || inputs[i].level == exclude_level) { + continue; + } + if (inputs[i].level == 0) { + // we need to consider all files on level 0 + for (const auto* f : inputs[i].files) { + if (!initialized || icmp->Compare(f->smallest, *smallest_key) < 0) { + *smallest_key = f->smallest; + } + if (!initialized || icmp->Compare(f->largest, *largest_key) > 0) { + *largest_key = f->largest; + } + initialized = true; + } + } else { + // we only need to consider the first and last file + if (!initialized || + icmp->Compare(inputs[i].files[0]->smallest, *smallest_key) < 0) { + *smallest_key = inputs[i].files[0]->smallest; + } + if (!initialized || + icmp->Compare(inputs[i].files.back()->largest, *largest_key) > 0) { + *largest_key = inputs[i].files.back()->largest; + } + initialized = true; + } + } +} + std::vector Compaction::PopulateWithAtomicBoundaries( VersionStorageInfo* vstorage, std::vector inputs) { const Comparator* ucmp = vstorage->InternalComparator()->user_comparator(); @@ -203,6 +240,38 @@ bool Compaction::IsFullCompaction( return num_files_in_compaction == total_num_files; } +Status Compaction::InitInputTableProperties() { + if (!input_table_properties_.empty()) { + return Status::OK(); + } + + Status s; + const ReadOptions read_options(Env::IOActivity::kCompaction); + assert(input_version_); + for (size_t i = 0; i < num_input_levels(); ++i) { + for (const FileMetaData* fmd : *(this->inputs(i))) { + std::shared_ptr tp; + std::string file_name = + TableFileName(immutable_options_.cf_paths, fmd->fd.GetNumber(), + fmd->fd.GetPathId()); + s = input_version_->GetTableProperties(read_options, &tp, fmd, + &file_name); + if (s.ok()) { + input_table_properties_[file_name] = tp; + } else { + ROCKS_LOG_ERROR(immutable_options_.info_log, + "Unable to load table properties for file %" PRIu64 + " --- %s\n", + fmd->fd.GetNumber(), s.ToString().c_str()); + input_table_properties_.clear(); + return s; + } + } + } + + return s; +} + Compaction::Compaction( VersionStorageInfo* vstorage, const ImmutableOptions& _immutable_options, const MutableCFOptions& _mutable_cf_options, @@ -366,9 +435,14 @@ void Compaction::PopulatePenultimateLevelOutputRange() { } } - GetBoundaryKeys(input_vstorage_, inputs_, - &penultimate_level_smallest_user_key_, - &penultimate_level_largest_user_key_, exclude_level); + // FIXME: should make use of `penultimate_output_range_type_`. + // FIXME: when last level's input range does not overlap with + // penultimate level, and penultimate level input is empty, + // this call will not set penultimate_level_smallest_ or + // penultimate_level_largest_. No keys will be compacted up. + GetBoundaryInternalKeys(input_vstorage_, inputs_, + &penultimate_level_smallest_, + &penultimate_level_largest_, exclude_level); } Compaction::~Compaction() { @@ -393,33 +467,40 @@ bool Compaction::OverlapPenultimateLevelOutputRange( if (!SupportsPerKeyPlacement()) { return false; } + + // See FIXME in Compaction::PopulatePenultimateLevelOutputRange(). + // We do not compact any key up in this case. + if (penultimate_level_smallest_.size() == 0 || + penultimate_level_largest_.size() == 0) { + return false; + } + const Comparator* ucmp = input_vstorage_->InternalComparator()->user_comparator(); return ucmp->CompareWithoutTimestamp( - smallest_key, penultimate_level_largest_user_key_) <= 0 && + smallest_key, penultimate_level_largest_.user_key()) <= 0 && ucmp->CompareWithoutTimestamp( - largest_key, penultimate_level_smallest_user_key_) >= 0; + largest_key, penultimate_level_smallest_.user_key()) >= 0; } // key includes timestamp if user-defined timestamp is enabled. -bool Compaction::WithinPenultimateLevelOutputRange(const Slice& key) const { +bool Compaction::WithinPenultimateLevelOutputRange( + const ParsedInternalKey& ikey) const { if (!SupportsPerKeyPlacement()) { return false; } - if (penultimate_level_smallest_user_key_.empty() || - penultimate_level_largest_user_key_.empty()) { + if (penultimate_level_smallest_.size() == 0 || + penultimate_level_largest_.size() == 0) { return false; } - const Comparator* ucmp = - input_vstorage_->InternalComparator()->user_comparator(); + const InternalKeyComparator* icmp = input_vstorage_->InternalComparator(); - return ucmp->CompareWithoutTimestamp( - key, penultimate_level_smallest_user_key_) >= 0 && - ucmp->CompareWithoutTimestamp( - key, penultimate_level_largest_user_key_) <= 0; + // op_type of a key can change during compaction, e.g. Merge -> Put. + return icmp->CompareKeySeq(ikey, penultimate_level_smallest_.Encode()) >= 0 && + icmp->CompareKeySeq(ikey, penultimate_level_largest_.Encode()) <= 0; } bool Compaction::InputCompressionMatchesOutput() const { @@ -745,8 +826,17 @@ std::unique_ptr Compaction::CreateCompactionFilter() const { CompactionFilter::Context context; context.is_full_compaction = is_full_compaction_; context.is_manual_compaction = is_manual_compaction_; + context.input_start_level = start_level_; context.column_family_id = cfd_->GetID(); context.reason = TableFileCreationReason::kCompaction; + context.input_table_properties = GetInputTableProperties(); + if (context.input_table_properties.empty()) { + ROCKS_LOG_WARN( + immutable_options_.info_log, + "Unable to set `input_table_properties` of `CompactionFilter::Context` " + "for compaction."); + } + return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter( context); } diff --git a/db/compaction/compaction.h b/db/compaction/compaction.h index 1bd406bc9..50c75f70b 100644 --- a/db/compaction/compaction.h +++ b/db/compaction/compaction.h @@ -289,7 +289,14 @@ class Compaction { // is the sum of all input file sizes. uint64_t OutputFilePreallocationSize() const; - void SetInputVersion(Version* input_version); + // TODO(hx235): eventually we should consider `InitInputTableProperties()`'s + // status and fail the compaction if needed + // TODO(hx235): consider making this function part of the construction so we + // don't forget to call it + void FinalizeInputInfo(Version* input_version) { + SetInputVersion(input_version); + InitInputTableProperties().PermitUncheckedError(); + } struct InputLevelSummaryBuffer { char buffer[128]; @@ -326,26 +333,26 @@ class Compaction { int output_level, VersionStorageInfo* vstorage, const std::vector& inputs); - TablePropertiesCollection GetOutputTableProperties() const { - return output_table_properties_; + const TablePropertiesCollection& GetInputTableProperties() const { + return input_table_properties_; } - void SetOutputTableProperties(TablePropertiesCollection tp) { - output_table_properties_ = std::move(tp); + // TODO(hx235): consider making this function symmetric to + // InitInputTableProperties() + void SetOutputTableProperties( + const std::string& file_name, + const std::shared_ptr& tp) { + output_table_properties_[file_name] = tp; + } + + const TablePropertiesCollection& GetOutputTableProperties() const { + return output_table_properties_; } Slice GetSmallestUserKey() const { return smallest_user_key_; } Slice GetLargestUserKey() const { return largest_user_key_; } - Slice GetPenultimateLevelSmallestUserKey() const { - return penultimate_level_smallest_user_key_; - } - - Slice GetPenultimateLevelLargestUserKey() const { - return penultimate_level_largest_user_key_; - } - PenultimateOutputRangeType GetPenultimateOutputRangeType() const { return penultimate_output_range_type_; } @@ -368,10 +375,8 @@ class Compaction { // per_key_placement feature, which is safe to place the key to the // penultimate level. different compaction strategy has different rules. // If per_key_placement is not supported, always return false. - // TODO: currently it doesn't support moving data from the last level to the - // penultimate level // key includes timestamp if user-defined timestamp is enabled. - bool WithinPenultimateLevelOutputRange(const Slice& key) const; + bool WithinPenultimateLevelOutputRange(const ParsedInternalKey& ikey) const; CompactionReason compaction_reason() const { return compaction_reason_; } @@ -428,6 +433,10 @@ class Compaction { const int output_level); private: + void SetInputVersion(Version* input_version); + + Status InitInputTableProperties(); + // mark (or clear) all files that are being compacted void MarkFilesBeingCompacted(bool mark_as_compacted); @@ -437,6 +446,13 @@ class Compaction { Slice* smallest_key, Slice* largest_key, int exclude_level = -1); + // get the smallest and largest internal key present in files to be compacted + static void GetBoundaryInternalKeys( + VersionStorageInfo* vstorage, + const std::vector& inputs, + InternalKey* smallest_key, InternalKey* largest_key, + int exclude_level = -1); + // populate penultimate level output range, which will be used to determine if // a key is safe to output to the penultimate level (details see // `Compaction::WithinPenultimateLevelOutputRange()`. @@ -518,7 +534,7 @@ class Compaction { // Does input compression match the output compression? bool InputCompressionMatchesOutput() const; - // table properties of output files + TablePropertiesCollection input_table_properties_; TablePropertiesCollection output_table_properties_; // smallest user keys in compaction @@ -549,8 +565,8 @@ class Compaction { // Key range for penultimate level output // includes timestamp if user-defined timestamp is enabled. // penultimate_output_range_type_ shows the range type - Slice penultimate_level_smallest_user_key_; - Slice penultimate_level_largest_user_key_; + InternalKey penultimate_level_smallest_; + InternalKey penultimate_level_largest_; PenultimateOutputRangeType penultimate_output_range_type_ = PenultimateOutputRangeType::kNotSupported; }; diff --git a/db/compaction/compaction_iterator.cc b/db/compaction/compaction_iterator.cc index 5be7b565a..85d1c039b 100644 --- a/db/compaction/compaction_iterator.cc +++ b/db/compaction/compaction_iterator.cc @@ -14,6 +14,7 @@ #include "db/blob/prefetch_buffer_collection.h" #include "db/snapshot_checker.h" #include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "logging/logging.h" #include "port/likely.h" #include "rocksdb/listener.h" @@ -31,7 +32,8 @@ CompactionIterator::CompactionIterator( BlobFileBuilder* blob_file_builder, bool allow_data_in_errors, bool enforce_single_del_contracts, const std::atomic& manual_compaction_canceled, - const Compaction* compaction, const CompactionFilter* compaction_filter, + bool must_count_input_entries, const Compaction* compaction, + const CompactionFilter* compaction_filter, const std::atomic* shutting_down, const std::shared_ptr info_log, const std::string* full_history_ts_low, @@ -45,8 +47,9 @@ CompactionIterator::CompactionIterator( manual_compaction_canceled, std::unique_ptr( compaction ? new RealCompaction(compaction) : nullptr), - compaction_filter, shutting_down, info_log, full_history_ts_low, - preserve_time_min_seqno, preclude_last_level_min_seqno) {} + must_count_input_entries, compaction_filter, shutting_down, info_log, + full_history_ts_low, preserve_time_min_seqno, + preclude_last_level_min_seqno) {} CompactionIterator::CompactionIterator( InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper, @@ -58,15 +61,14 @@ CompactionIterator::CompactionIterator( BlobFileBuilder* blob_file_builder, bool allow_data_in_errors, bool enforce_single_del_contracts, const std::atomic& manual_compaction_canceled, - std::unique_ptr compaction, + std::unique_ptr compaction, bool must_count_input_entries, const CompactionFilter* compaction_filter, const std::atomic* shutting_down, const std::shared_ptr info_log, const std::string* full_history_ts_low, const SequenceNumber preserve_time_min_seqno, const SequenceNumber preclude_last_level_min_seqno) - : input_(input, cmp, - !compaction || compaction->DoesInputReferenceBlobFiles()), + : input_(input, cmp, must_count_input_entries), cmp_(cmp), merge_helper_(merge_helper), snapshots_(snapshots), @@ -422,16 +424,13 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip, return false; } else if (decision == CompactionFilter::Decision::kChangeWideColumnEntity) { WideColumns sorted_columns; - sorted_columns.reserve(new_columns.size()); + for (const auto& column : new_columns) { sorted_columns.emplace_back(column.first, column.second); } - std::sort(sorted_columns.begin(), sorted_columns.end(), - [](const WideColumn& lhs, const WideColumn& rhs) { - return lhs.name().compare(rhs.name()) < 0; - }); + WideColumnsHelper::SortColumns(sorted_columns); { const Status s = WideColumnSerialization::Serialize( @@ -1228,7 +1227,7 @@ void CompactionIterator::DecideOutputLevel() { // not from this compaction. // TODO: add statistic for declined output_to_penultimate_level bool safe_to_penultimate_level = - compaction_->WithinPenultimateLevelOutputRange(ikey_.user_key); + compaction_->WithinPenultimateLevelOutputRange(ikey_); if (!safe_to_penultimate_level) { output_to_penultimate_level_ = false; // It could happen when disable/enable `last_level_temperature` while @@ -1257,10 +1256,13 @@ void CompactionIterator::PrepareOutput() { } else if (ikey_.type == kTypeBlobIndex) { GarbageCollectBlobIfNeeded(); } - } - if (compaction_ != nullptr && compaction_->SupportsPerKeyPlacement()) { - DecideOutputLevel(); + // For range del sentinel, we don't use it to cut files for bottommost + // compaction. So it should not make a difference which output level we + // decide. + if (compaction_ != nullptr && compaction_->SupportsPerKeyPlacement()) { + DecideOutputLevel(); + } } // Zeroing out the sequence number leads to better compression. diff --git a/db/compaction/compaction_iterator.h b/db/compaction/compaction_iterator.h index ea2dc062e..1ff9c8869 100644 --- a/db/compaction/compaction_iterator.h +++ b/db/compaction/compaction_iterator.h @@ -38,15 +38,18 @@ class SequenceIterWrapper : public InternalIterator { bool Valid() const override { return inner_iter_->Valid(); } Status status() const override { return inner_iter_->status(); } void Next() override { - num_itered_++; + if (!inner_iter_->IsDeleteRangeSentinelKey()) { + num_itered_++; + } inner_iter_->Next(); } void Seek(const Slice& target) override { if (!need_count_entries_) { + has_num_itered_ = false; inner_iter_->Seek(target); } else { - // For flush cases, we need to count total number of entries, so we - // do Next() rather than Seek(). + // Need to count total number of entries, + // so we do Next() rather than Seek(). while (inner_iter_->Valid() && icmp_.Compare(inner_iter_->key(), target) < 0) { Next(); @@ -62,7 +65,8 @@ class SequenceIterWrapper : public InternalIterator { void SeekForPrev(const Slice& /* target */) override { assert(false); } void SeekToLast() override { assert(false); } - uint64_t num_itered() const { return num_itered_; } + uint64_t NumItered() const { return num_itered_; } + bool HasNumItered() const { return has_num_itered_; } bool IsDeleteRangeSentinelKey() const override { assert(Valid()); return inner_iter_->IsDeleteRangeSentinelKey(); @@ -73,6 +77,7 @@ class SequenceIterWrapper : public InternalIterator { InternalIterator* inner_iter_; // not owned uint64_t num_itered_ = 0; bool need_count_entries_; + bool has_num_itered_ = true; }; class CompactionIterator { @@ -114,7 +119,8 @@ class CompactionIterator { virtual bool SupportsPerKeyPlacement() const = 0; // `key` includes timestamp if user-defined timestamp is enabled. - virtual bool WithinPenultimateLevelOutputRange(const Slice& key) const = 0; + virtual bool WithinPenultimateLevelOutputRange( + const ParsedInternalKey&) const = 0; }; class RealCompaction : public CompactionProxy { @@ -181,14 +187,19 @@ class CompactionIterator { // Check if key is within penultimate level output range, to see if it's // safe to output to the penultimate level for per_key_placement feature. // `key` includes timestamp if user-defined timestamp is enabled. - bool WithinPenultimateLevelOutputRange(const Slice& key) const override { - return compaction_->WithinPenultimateLevelOutputRange(key); + bool WithinPenultimateLevelOutputRange( + const ParsedInternalKey& ikey) const override { + return compaction_->WithinPenultimateLevelOutputRange(ikey); } private: const Compaction* compaction_; }; + // @param must_count_input_entries if true, `NumInputEntryScanned()` will + // return the number of input keys scanned. If false, `NumInputEntryScanned()` + // will return this number if no Seek was called on `input`. User should call + // `HasNumInputEntryScanned()` first in this case. CompactionIterator( InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper, SequenceNumber last_sequence, std::vector* snapshots, @@ -199,7 +210,7 @@ class CompactionIterator { BlobFileBuilder* blob_file_builder, bool allow_data_in_errors, bool enforce_single_del_contracts, const std::atomic& manual_compaction_canceled, - const Compaction* compaction = nullptr, + bool must_count_input_entries, const Compaction* compaction = nullptr, const CompactionFilter* compaction_filter = nullptr, const std::atomic* shutting_down = nullptr, const std::shared_ptr info_log = nullptr, @@ -219,6 +230,7 @@ class CompactionIterator { bool enforce_single_del_contracts, const std::atomic& manual_compaction_canceled, std::unique_ptr compaction, + bool must_count_input_entries, const CompactionFilter* compaction_filter = nullptr, const std::atomic* shutting_down = nullptr, const std::shared_ptr info_log = nullptr, @@ -253,7 +265,8 @@ class CompactionIterator { return current_user_key_; } const CompactionIterationStats& iter_stats() const { return iter_stats_; } - uint64_t num_input_entry_scanned() const { return input_.num_itered(); } + bool HasNumInputEntryScanned() const { return input_.HasNumItered(); } + uint64_t NumInputEntryScanned() const { return input_.NumItered(); } // If the current key should be placed on penultimate level, only valid if // per_key_placement is supported bool output_to_penultimate_level() const { diff --git a/db/compaction/compaction_iterator_test.cc b/db/compaction/compaction_iterator_test.cc index 81362d792..699e62969 100644 --- a/db/compaction/compaction_iterator_test.cc +++ b/db/compaction/compaction_iterator_test.cc @@ -184,8 +184,9 @@ class FakeCompaction : public CompactionIterator::CompactionProxy { return supports_per_key_placement; } - bool WithinPenultimateLevelOutputRange(const Slice& key) const override { - return (!key.starts_with("unsafe_pb")); + bool WithinPenultimateLevelOutputRange( + const ParsedInternalKey& key) const override { + return (!key.user_key.starts_with("unsafe_pb")); } bool key_not_exists_beyond_output_level = false; @@ -293,8 +294,8 @@ class CompactionIteratorTest : public testing::TestWithParam { nullptr /* blob_file_builder */, true /*allow_data_in_errors*/, true /*enforce_single_del_contracts*/, /*manual_compaction_canceled=*/kManualCompactionCanceledFalse_, - std::move(compaction), filter, &shutting_down_, /*info_log=*/nullptr, - full_history_ts_low)); + std::move(compaction), /*must_count_input_entries=*/false, filter, + &shutting_down_, /*info_log=*/nullptr, full_history_ts_low)); } void AddSnapshot(SequenceNumber snapshot, diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index d609e0154..99b099759 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -288,23 +288,23 @@ void CompactionJob::Prepare() { if (preserve_time_duration > 0) { const ReadOptions read_options(Env::IOActivity::kCompaction); - // setup seqno_time_mapping_ - seqno_time_mapping_.SetMaxTimeDuration(preserve_time_duration); + // setup seqno_to_time_mapping_ + seqno_to_time_mapping_.SetMaxTimeDuration(preserve_time_duration); for (const auto& each_level : *c->inputs()) { for (const auto& fmd : each_level.files) { std::shared_ptr tp; Status s = cfd->current()->GetTableProperties(read_options, &tp, fmd, nullptr); if (s.ok()) { - seqno_time_mapping_.Add(tp->seqno_to_time_mapping) + seqno_to_time_mapping_.Add(tp->seqno_to_time_mapping) .PermitUncheckedError(); - seqno_time_mapping_.Add(fmd->fd.smallest_seqno, - fmd->oldest_ancester_time); + seqno_to_time_mapping_.Add(fmd->fd.smallest_seqno, + fmd->oldest_ancester_time); } } } - auto status = seqno_time_mapping_.Sort(); + auto status = seqno_to_time_mapping_.Sort(); if (!status.ok()) { ROCKS_LOG_WARN(db_options_.info_log, "Invalid sequence number to time mapping: Status: %s", @@ -320,13 +320,17 @@ void CompactionJob::Prepare() { preserve_time_min_seqno_ = 0; preclude_last_level_min_seqno_ = 0; } else { - seqno_time_mapping_.TruncateOldEntries(_current_time); + seqno_to_time_mapping_.TruncateOldEntries(_current_time); uint64_t preserve_time = static_cast(_current_time) > preserve_time_duration ? _current_time - preserve_time_duration : 0; + // GetProximalSeqnoBeforeTime tells us the last seqno known to have been + // written at or before the given time. + 1 to get the minimum we should + // preserve without excluding anything that might have been written on or + // after the given time. preserve_time_min_seqno_ = - seqno_time_mapping_.GetOldestSequenceNum(preserve_time); + seqno_to_time_mapping_.GetProximalSeqnoBeforeTime(preserve_time) + 1; if (c->immutable_options()->preclude_last_level_data_seconds > 0) { uint64_t preclude_last_level_time = static_cast(_current_time) > @@ -335,7 +339,9 @@ void CompactionJob::Prepare() { c->immutable_options()->preclude_last_level_data_seconds : 0; preclude_last_level_min_seqno_ = - seqno_time_mapping_.GetOldestSequenceNum(preclude_last_level_time); + seqno_to_time_mapping_.GetProximalSeqnoBeforeTime( + preclude_last_level_time) + + 1; } } } @@ -796,15 +802,46 @@ Status CompactionJob::Run() { auto fn = TableFileName(state.compaction->immutable_options()->cf_paths, output.meta.fd.GetNumber(), output.meta.fd.GetPathId()); - tp[fn] = output.table_properties; + compact_->compaction->SetOutputTableProperties(fn, + output.table_properties); } } - compact_->compaction->SetOutputTableProperties(std::move(tp)); - // Finish up all book-keeping to unify the subcompaction results + // Finish up all bookkeeping to unify the subcompaction results. compact_->AggregateCompactionStats(compaction_stats_, *compaction_job_stats_); - UpdateCompactionStats(); - + uint64_t num_input_range_del = 0; + bool ok = UpdateCompactionStats(&num_input_range_del); + // (Sub)compactions returned ok, do sanity check on the number of input keys. + if (status.ok() && ok && compaction_job_stats_->has_num_input_records) { + size_t ts_sz = compact_->compaction->column_family_data() + ->user_comparator() + ->timestamp_size(); + // When trim_ts_ is non-empty, CompactionIterator takes + // HistoryTrimmingIterator as input iterator and sees a trimmed view of + // input keys. So the number of keys it processed is not suitable for + // verification here. + // TODO: support verification when trim_ts_ is non-empty. + if (!(ts_sz > 0 && !trim_ts_.empty()) && + db_options_.compaction_verify_record_count) { + assert(compaction_stats_.stats.num_input_records > 0); + // TODO: verify the number of range deletion entries. + uint64_t expected = + compaction_stats_.stats.num_input_records - num_input_range_del; + uint64_t actual = compaction_job_stats_->num_input_records; + if (expected != actual) { + std::string msg = + "Total number of input records: " + std::to_string(expected) + + ", but processed " + std::to_string(actual) + " records."; + ROCKS_LOG_WARN( + db_options_.info_log, "[%s] [JOB %d] Compaction %s", + compact_->compaction->column_family_data()->GetName().c_str(), + job_context_->job_id, msg.c_str()); + status = Status::Corruption( + "Compaction number of input keys does not match number of keys " + "processed."); + } + } + } RecordCompactionIOStats(); LogFlush(db_options_.info_log); TEST_SYNC_POINT("CompactionJob::Run():End"); @@ -813,7 +850,8 @@ Status CompactionJob::Run() { return status; } -Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) { +Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options, + bool* compaction_released) { assert(compact_); AutoThreadOperationStageUpdater stage_updater( @@ -829,7 +867,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) { compaction_stats_); if (status.ok()) { - status = InstallCompactionResults(mutable_cf_options); + status = InstallCompactionResults(mutable_cf_options, compaction_released); } if (!versions_->io_status().ok()) { io_status_ = versions_->io_status(); @@ -1252,6 +1290,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { /*expect_valid_internal_key=*/true, range_del_agg.get(), blob_file_builder.get(), db_options_.allow_data_in_errors, db_options_.enforce_single_del_contracts, manual_compaction_canceled_, + sub_compact->compaction + ->DoesInputReferenceBlobFiles() /* must_count_input_entries */, sub_compact->compaction, compaction_filter, shutting_down_, db_options_.info_log, full_history_ts_low, preserve_time_min_seqno_, preclude_last_level_min_seqno_); @@ -1285,6 +1325,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { "CompactionJob::ProcessKeyValueCompaction()::Processing", reinterpret_cast( const_cast(sub_compact->compaction))); + uint64_t last_cpu_micros = prev_cpu_micros; while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) { // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid() // returns true. @@ -1296,6 +1337,12 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats); c_iter->ResetRecordCounts(); RecordCompactionIOStats(); + + uint64_t cur_cpu_micros = db_options_.clock->CPUMicros(); + assert(cur_cpu_micros >= last_cpu_micros); + RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME, + cur_cpu_micros - last_cpu_micros); + last_cpu_micros = cur_cpu_micros; } // Add current compaction_iterator key to target compaction output, if the @@ -1316,8 +1363,25 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { if (c_iter->status().IsManualCompactionPaused()) { break; } + +#ifndef NDEBUG + bool stop = false; + TEST_SYNC_POINT_CALLBACK("CompactionJob::ProcessKeyValueCompaction()::stop", + static_cast(&stop)); + if (stop) { + break; + } +#endif // NDEBUG } + // This number may not be accurate when CompactionIterator was created + // with `must_count_input_entries=false`. + assert(!sub_compact->compaction->DoesInputReferenceBlobFiles() || + c_iter->HasNumInputEntryScanned()); + sub_compact->compaction_job_stats.has_num_input_records = + c_iter->HasNumInputEntryScanned(); + sub_compact->compaction_job_stats.num_input_records = + c_iter->NumInputEntryScanned(); sub_compact->compaction_job_stats.num_blobs_read = c_iter_stats.num_blobs_read; sub_compact->compaction_job_stats.total_blob_bytes_read = @@ -1386,8 +1450,11 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { sub_compact->Current().UpdateBlobStats(); } + uint64_t cur_cpu_micros = db_options_.clock->CPUMicros(); sub_compact->compaction_job_stats.cpu_micros = - db_options_.clock->CPUMicros() - prev_cpu_micros; + cur_cpu_micros - prev_cpu_micros; + RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME, + cur_cpu_micros - last_cpu_micros); if (measure_io_stats_) { sub_compact->compaction_job_stats.file_write_nanos += @@ -1509,7 +1576,7 @@ Status CompactionJob::FinishCompactionOutputFile( const uint64_t current_entries = outputs.NumEntries(); - s = outputs.Finish(s, seqno_time_mapping_); + s = outputs.Finish(s, seqno_to_time_mapping_); if (s.ok()) { // With accurate smallest and largest key, we can get a slightly more @@ -1637,7 +1704,7 @@ Status CompactionJob::FinishCompactionOutputFile( } Status CompactionJob::InstallCompactionResults( - const MutableCFOptions& mutable_cf_options) { + const MutableCFOptions& mutable_cf_options, bool* compaction_released) { assert(compact_); db_mutex_->AssertHeld(); @@ -1719,9 +1786,15 @@ Status CompactionJob::InstallCompactionResults( } } - return versions_->LogAndApply(compaction->column_family_data(), - mutable_cf_options, read_options, edit, - db_mutex_, db_directory_); + auto manifest_wcb = [&compaction, &compaction_released](const Status& s) { + compaction->ReleaseCompactionFiles(s); + *compaction_released = true; + }; + + return versions_->LogAndApply( + compaction->column_family_data(), mutable_cf_options, read_options, edit, + db_mutex_, db_directory_, /*new_descriptor_log=*/false, + /*column_family_options=*/nullptr, manifest_wcb); } void CompactionJob::RecordCompactionIOStats() { @@ -1827,6 +1900,8 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact, sub_compact->start.has_value() ? &tmp_start : nullptr, sub_compact->end.has_value() ? &tmp_end : nullptr); if (oldest_ancester_time == std::numeric_limits::max()) { + // TODO: fix DBSSTTest.GetTotalSstFilesSize and use + // kUnknownOldestAncesterTime oldest_ancester_time = current_time; } @@ -1871,6 +1946,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact, db_options_.stats, listeners, db_options_.file_checksum_gen_factory.get(), tmp_set.Contains(FileType::kTableFile), false)); + // TODO(hx235): pass in the correct `oldest_key_time` instead of `0` TableBuilderOptions tboptions( *cfd->ioptions(), *(sub_compact->compaction->mutable_cf_options()), cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(), @@ -1903,24 +1979,53 @@ void CopyPrefix(const Slice& src, size_t prefix_length, std::string* dst) { } } // namespace - -void CompactionJob::UpdateCompactionStats() { +bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) { assert(compact_); Compaction* compaction = compact_->compaction; compaction_stats_.stats.num_input_files_in_non_output_levels = 0; compaction_stats_.stats.num_input_files_in_output_level = 0; + + bool has_error = false; + const ReadOptions read_options(Env::IOActivity::kCompaction); + const auto& input_table_properties = compaction->GetInputTableProperties(); for (int input_level = 0; input_level < static_cast(compaction->num_input_levels()); ++input_level) { + size_t num_input_files = compaction->num_input_files(input_level); + uint64_t* bytes_read; if (compaction->level(input_level) != compaction->output_level()) { - UpdateCompactionInputStatsHelper( - &compaction_stats_.stats.num_input_files_in_non_output_levels, - &compaction_stats_.stats.bytes_read_non_output_levels, input_level); + compaction_stats_.stats.num_input_files_in_non_output_levels += + static_cast(num_input_files); + bytes_read = &compaction_stats_.stats.bytes_read_non_output_levels; } else { - UpdateCompactionInputStatsHelper( - &compaction_stats_.stats.num_input_files_in_output_level, - &compaction_stats_.stats.bytes_read_output_level, input_level); + compaction_stats_.stats.num_input_files_in_output_level += + static_cast(num_input_files); + bytes_read = &compaction_stats_.stats.bytes_read_output_level; + } + for (size_t i = 0; i < num_input_files; ++i) { + const FileMetaData* file_meta = compaction->input(input_level, i); + *bytes_read += file_meta->fd.GetFileSize(); + uint64_t file_input_entries = file_meta->num_entries; + uint64_t file_num_range_del = file_meta->num_range_deletions; + if (file_input_entries == 0) { + uint64_t file_number = file_meta->fd.GetNumber(); + // Try getting info from table property + std::string fn = + TableFileName(compaction->immutable_options()->cf_paths, + file_number, file_meta->fd.GetPathId()); + const auto& tp = input_table_properties.find(fn); + if (tp != input_table_properties.end()) { + file_input_entries = tp->second->num_entries; + file_num_range_del = tp->second->num_range_deletions; + } else { + has_error = true; + } + } + compaction_stats_.stats.num_input_records += file_input_entries; + if (num_input_range_del) { + *num_input_range_del += file_num_range_del; + } } } @@ -1930,21 +2035,7 @@ void CompactionJob::UpdateCompactionStats() { compaction_stats_.stats.num_dropped_records = compaction_stats_.DroppedRecords(); -} - -void CompactionJob::UpdateCompactionInputStatsHelper(int* num_files, - uint64_t* bytes_read, - int input_level) { - const Compaction* compaction = compact_->compaction; - auto num_input_files = compaction->num_input_files(input_level); - *num_files += static_cast(num_input_files); - - for (size_t i = 0; i < num_input_files; ++i) { - const auto* file_meta = compaction->input(input_level, i); - *bytes_read += file_meta->fd.GetFileSize(); - compaction_stats_.stats.num_input_records += - static_cast(file_meta->num_entries); - } + return !has_error; } void CompactionJob::UpdateCompactionJobStats( diff --git a/db/compaction/compaction_job.h b/db/compaction/compaction_job.h index a930c15f1..e812cfc72 100644 --- a/db/compaction/compaction_job.h +++ b/db/compaction/compaction_job.h @@ -186,13 +186,30 @@ class CompactionJob { // REQUIRED: mutex held // Add compaction input/output to the current version - Status Install(const MutableCFOptions& mutable_cf_options); + // Releases compaction file through Compaction::ReleaseCompactionFiles(). + // Sets *compaction_released to true if compaction is released. + Status Install(const MutableCFOptions& mutable_cf_options, + bool* compaction_released); // Return the IO status IOStatus io_status() const { return io_status_; } protected: - void UpdateCompactionStats(); + // Update the following stats in compaction_stats_.stats + // - num_input_files_in_non_output_levels + // - num_input_files_in_output_level + // - bytes_read_non_output_levels + // - bytes_read_output_level + // - num_input_records + // - bytes_read_blob + // - num_dropped_records + // + // @param num_input_range_del if non-null, will be set to the number of range + // deletion entries in this compaction input. + // + // Returns true iff compaction_stats_.stats.num_input_records and + // num_input_range_del are calculated successfully. + bool UpdateCompactionStats(uint64_t* num_input_range_del = nullptr); void LogCompaction(); virtual void RecordCompactionIOStats(); void CleanupCompaction(); @@ -259,7 +276,8 @@ class CompactionJob { const Slice& next_table_min_key, const Slice* comp_start_user_key, const Slice* comp_end_user_key); - Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options); + Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options, + bool* compaction_released); Status OpenCompactionOutputFile(SubcompactionState* sub_compact, CompactionOutputs& outputs); void UpdateCompactionJobStats( @@ -267,9 +285,6 @@ class CompactionJob { void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats, CompactionJobStats* compaction_job_stats = nullptr); - void UpdateCompactionInputStatsHelper(int* num_files, uint64_t* bytes_read, - int input_level); - void NotifyOnSubcompactionBegin(SubcompactionState* sub_compact); void NotifyOnSubcompactionCompleted(SubcompactionState* sub_compact); @@ -335,7 +350,7 @@ class CompactionJob { // Stores the sequence number to time mapping gathered from all input files // it also collects the smallest_seqno -> oldest_ancester_time from the SST. - SeqnoToTimeMapping seqno_time_mapping_; + SeqnoToTimeMapping seqno_to_time_mapping_; // Minimal sequence number for preserving the time information. The time info // older than this sequence number won't be preserved after the compaction and diff --git a/db/compaction/compaction_job_test.cc b/db/compaction/compaction_job_test.cc index f7fc28c15..a16891110 100644 --- a/db/compaction/compaction_job_test.cc +++ b/db/compaction/compaction_job_test.cc @@ -215,7 +215,9 @@ class CompactionJobTestBase : public testing::Test { dbname_, &db_options_, env_options_, table_cache_.get(), &write_buffer_manager_, &write_controller_, /*block_cache_tracer=*/nullptr, - /*io_tracer=*/nullptr, /*db_id*/ "", /*db_session_id*/ "")), + /*io_tracer=*/nullptr, /*db_id=*/"", /*db_session_id=*/"", + /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)), shutting_down_(false), mock_table_factory_(new mock::MockTableFactory()), error_handler_(nullptr, db_options_, &mutex_), @@ -540,11 +542,12 @@ class CompactionJobTestBase : public testing::Test { ASSERT_OK(s); db_options_.info_log = info_log; - versions_.reset( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + versions_.reset(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); compaction_job_stats_.Reset(); ASSERT_OK(SetIdentityFile(env_, dbname_)); @@ -644,7 +647,7 @@ class CompactionJobTestBase : public testing::Test { mutable_cf_options_.max_compaction_bytes, 0, kNoCompression, cfd->GetLatestMutableCFOptions()->compression_opts, Temperature::kUnknown, max_subcompactions, grandparents, true); - compaction.SetInputVersion(cfd->current()); + compaction.FinalizeInputInfo(cfd->current()); assert(db_options_.info_log); LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get()); @@ -655,11 +658,12 @@ class CompactionJobTestBase : public testing::Test { ASSERT_TRUE(full_history_ts_low_.empty() || ucmp_->timestamp_size() == full_history_ts_low_.size()); const std::atomic kManualCompactionCanceledFalse{false}; + JobContext job_context(1, false /* create_superversion */); CompactionJob compaction_job( 0, &compaction, db_options_, mutable_db_options_, env_options_, versions_.get(), &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, nullptr, &mutex_, &error_handler_, snapshots, - earliest_write_conflict_snapshot, snapshot_checker, nullptr, + earliest_write_conflict_snapshot, snapshot_checker, &job_context, table_cache_, &event_logger, false, false, dbname_, &compaction_job_stats_, Env::Priority::USER, nullptr /* IOTracer */, /*manual_compaction_canceled=*/kManualCompactionCanceledFalse, @@ -673,7 +677,9 @@ class CompactionJobTestBase : public testing::Test { ASSERT_OK(s); ASSERT_OK(compaction_job.io_status()); mutex_.Lock(); - ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions())); + bool compaction_released = false; + ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(), + &compaction_released)); ASSERT_OK(compaction_job.io_status()); mutex_.Unlock(); log_buffer.FlushBufferToLog(); @@ -1521,13 +1527,15 @@ TEST_F(CompactionJobTest, VerifyPenultimateLevelOutput) { {files0, files1, files2, files3}, input_levels, /*verify_func=*/[&](Compaction& comp) { for (char c = 'a'; c <= 'z'; c++) { - std::string c_str; - c_str = c; - const Slice key(c_str); if (c == 'a') { - ASSERT_FALSE(comp.WithinPenultimateLevelOutputRange(key)); + ParsedInternalKey pik("a", 0U, kTypeValue); + ASSERT_FALSE(comp.WithinPenultimateLevelOutputRange(pik)); } else { - ASSERT_TRUE(comp.WithinPenultimateLevelOutputRange(key)); + std::string c_str{c}; + // WithinPenultimateLevelOutputRange checks internal key range. + // 'z' is the last key, so set seqno properly. + ParsedInternalKey pik(c_str, c == 'z' ? 12U : 0U, kTypeValue); + ASSERT_TRUE(comp.WithinPenultimateLevelOutputRange(pik)); } } }); diff --git a/db/compaction/compaction_outputs.cc b/db/compaction/compaction_outputs.cc index 3e21484c4..eb76cd849 100644 --- a/db/compaction/compaction_outputs.cc +++ b/db/compaction/compaction_outputs.cc @@ -18,16 +18,18 @@ void CompactionOutputs::NewBuilder(const TableBuilderOptions& tboptions) { builder_.reset(NewTableBuilder(tboptions, file_writer_.get())); } -Status CompactionOutputs::Finish(const Status& intput_status, - const SeqnoToTimeMapping& seqno_time_mapping) { +Status CompactionOutputs::Finish( + const Status& intput_status, + const SeqnoToTimeMapping& seqno_to_time_mapping) { FileMetaData* meta = GetMetaData(); assert(meta != nullptr); Status s = intput_status; if (s.ok()) { - std::string seqno_time_mapping_str; - seqno_time_mapping.Encode(seqno_time_mapping_str, meta->fd.smallest_seqno, - meta->fd.largest_seqno, meta->file_creation_time); - builder_->SetSeqnoTimeTableProperties(seqno_time_mapping_str, + std::string seqno_to_time_mapping_str; + seqno_to_time_mapping.Encode( + seqno_to_time_mapping_str, meta->fd.smallest_seqno, + meta->fd.largest_seqno, meta->file_creation_time); + builder_->SetSeqnoTimeTableProperties(seqno_to_time_mapping_str, meta->oldest_ancester_time); s = builder_->Finish(); diff --git a/db/compaction/compaction_outputs.h b/db/compaction/compaction_outputs.h index 6c3e3b6b3..18246cf2f 100644 --- a/db/compaction/compaction_outputs.h +++ b/db/compaction/compaction_outputs.h @@ -107,7 +107,7 @@ class CompactionOutputs { // Finish the current output file Status Finish(const Status& intput_status, - const SeqnoToTimeMapping& seqno_time_mapping); + const SeqnoToTimeMapping& seqno_to_time_mapping); // Update output table properties from table builder void UpdateTableProperties() { diff --git a/db/compaction/compaction_picker_fifo.cc b/db/compaction/compaction_picker_fifo.cc index 9aa24302e..505297770 100644 --- a/db/compaction/compaction_picker_fifo.cc +++ b/db/compaction/compaction_picker_fifo.cc @@ -17,6 +17,9 @@ #include "logging/log_buffer.h" #include "logging/logging.h" #include "options/options_helper.h" +#include "rocksdb/listener.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { diff --git a/db/compaction/compaction_picker_test.cc b/db/compaction/compaction_picker_test.cc index 6aec03840..3241d034d 100644 --- a/db/compaction/compaction_picker_test.cc +++ b/db/compaction/compaction_picker_test.cc @@ -84,7 +84,9 @@ class CompactionPickerTestBase : public testing::Test { options_.num_levels = num_levels; vstorage_.reset(new VersionStorageInfo( &icmp_, ucmp_, options_.num_levels, style, nullptr, false, - EpochNumberRequirement::kMustPresent)); + EpochNumberRequirement::kMustPresent, ioptions_.clock, + options_.bottommost_file_compaction_delay, + OffpeakTimeOption(mutable_db_options_.daily_offpeak_time_utc))); vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_); } @@ -93,7 +95,9 @@ class CompactionPickerTestBase : public testing::Test { void AddVersionStorage() { temp_vstorage_.reset(new VersionStorageInfo( &icmp_, ucmp_, options_.num_levels, ioptions_.compaction_style, - vstorage_.get(), false, EpochNumberRequirement::kMustPresent)); + vstorage_.get(), false, EpochNumberRequirement::kMustPresent, + ioptions_.clock, options_.bottommost_file_compaction_delay, + OffpeakTimeOption(mutable_db_options_.daily_offpeak_time_utc))); } void DeleteVersionStorage() { @@ -992,6 +996,61 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace5) { ASSERT_EQ(13, compaction->num_input_files(1)); } +TEST_F(CompactionPickerTest, + PartiallyExcludeL0ToReduceWriteStopForSizeAmpCompaction) { + const uint64_t kFileSize = 100000; + const uint64_t kL0FileCount = 30; + const uint64_t kLastLevelFileCount = 1; + const uint64_t kNumLevels = 5; + + for (const uint64_t test_no_exclusion : {false, true}) { + const uint64_t kExpectedNumExcludedL0 = + test_no_exclusion ? 0 : kL0FileCount * 1 / 10; + + mutable_cf_options_.level0_stop_writes_trigger = 36; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 1; + mutable_cf_options_.compaction_options_universal.max_merge_width = + test_no_exclusion + // In universal compaction, sorted runs from non L0 levels are + // counted toward `level0_stop_writes_trigger`. Therefore we need to + // subtract the total number of sorted runs picked originally for + // this compaction (i.e, kL0FileCount + kLastLevelFileCount) from + // `level0_stop_writes_trigger` to calculate `max_merge_width` that + // results in no L0 exclusion for testing purpose. + ? mutable_cf_options_.level0_stop_writes_trigger - + (kL0FileCount + kLastLevelFileCount) + : UINT_MAX; + + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(kNumLevels, kCompactionStyleUniversal); + + for (uint64_t i = 1; i <= kL0FileCount + kLastLevelFileCount; ++i) { + Add(i <= kL0FileCount ? 0 : kNumLevels - 1, static_cast(i), + std::to_string((i + 100) * 1000).c_str(), + std::to_string((i + 100) * 1000 + 999).c_str(), kFileSize, 0, i * 100, + i * 100 + 99); + } + + UpdateVersionStorageInfo(); + + ASSERT_TRUE(universal_compaction_picker.NeedsCompaction(vstorage_.get())); + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kUniversalSizeAmplification); + ASSERT_EQ(compaction->num_input_files(0), + kL0FileCount - kExpectedNumExcludedL0); + ASSERT_EQ(compaction->num_input_files(kNumLevels - 1), kLastLevelFileCount); + for (uint64_t level = 1; level <= kNumLevels - 2; level++) { + ASSERT_EQ(compaction->num_input_files(level), 0); + } + } +} + TEST_F(CompactionPickerTest, NeedsCompactionFIFO) { NewVersionStorage(1, kCompactionStyleFIFO); const int kFileCount = @@ -3378,6 +3437,9 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNonLastLevel) { ioptions_.preclude_last_level_data_seconds = 1000; mutable_cf_options_.compaction_options_universal .max_size_amplification_percent = 200; + // To avoid any L0 file exclusion in size amp compaction intended for reducing + // write stop + mutable_cf_options_.compaction_options_universal.max_merge_width = 2; UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); NewVersionStorage(kNumLevels, kCompactionStyleUniversal); @@ -3451,6 +3513,9 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNotSuport) { ioptions_.preclude_last_level_data_seconds = 1000; mutable_cf_options_.compaction_options_universal .max_size_amplification_percent = 200; + // To avoid any L0 file exclusion in size amp compaction intended for reducing + // write stop + mutable_cf_options_.compaction_options_universal.max_merge_width = 2; UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); NewVersionStorage(kNumLevels, kCompactionStyleUniversal); diff --git a/db/compaction/compaction_picker_universal.cc b/db/compaction/compaction_picker_universal.cc index 9eaf39546..6d9ff43cd 100644 --- a/db/compaction/compaction_picker_universal.cc +++ b/db/compaction/compaction_picker_universal.cc @@ -9,7 +9,7 @@ #include "db/compaction/compaction_picker_universal.h" -#include +#include #include #include #include @@ -114,6 +114,13 @@ class UniversalCompactionBuilder { // because some files are being compacted. Compaction* PickPeriodicCompaction(); + bool ShouldSkipLastSortedRunForSizeAmpCompaction() const { + assert(!sorted_runs_.empty()); + return ioptions_.preclude_last_level_data_seconds > 0 && + ioptions_.num_levels > 2 && + sorted_runs_.back().level == ioptions_.num_levels - 1 && + sorted_runs_.size() > 1; + } // Used in universal compaction when the allow_trivial_move // option is set. Checks whether there are any overlapping files // in the input. Returns true if the input files are non @@ -122,6 +129,100 @@ class UniversalCompactionBuilder { uint64_t GetMaxOverlappingBytes() const; + // To conditionally exclude some of the newest L0 files + // from a size amp compaction. This is to prevent a large number of L0 + // files from being locked by a size amp compaction, potentially leading to + // write stop with a few more flushes. + // + // Such exclusion is based on `num_l0_input_pre_exclusion`, + // `level0_stop_writes_trigger`, `max/min_merge_width` and the pre-exclusion + // compaction score. Noted that it will not make the size amp compaction of + // interest invalid from running as a size amp compaction as long as its + // pre-exclusion compaction score satisfies the condition to run. + // + // @param `num_l0_input_pre_exclusion` Number of L0 input files prior to + // exclusion + // @param `end_index` Index of the last sorted run selected as compaction + // input. Will not be affected by this exclusion. + // @param `start_index` Index of the first input sorted run prior to + // exclusion. Will be modified as output based on the exclusion. + // @param `candidate_size` Total size of all except for the last input sorted + // runs prior to exclusion. Will be modified as output based on the exclusion. + // + // @return Number of L0 files to exclude. `start_index` and + // `candidate_size` will be modified accordingly + std::size_t MightExcludeNewL0sToReduceWriteStop( + std::size_t num_l0_input_pre_exclusion, std::size_t end_index, + std::size_t& start_index, uint64_t& candidate_size) const { + if (num_l0_input_pre_exclusion == 0) { + return 0; + } + + assert(start_index <= end_index && sorted_runs_.size() > end_index); + assert(mutable_cf_options_.level0_stop_writes_trigger > 0); + const std::size_t level0_stop_writes_trigger = static_cast( + mutable_cf_options_.level0_stop_writes_trigger); + const std::size_t max_merge_width = static_cast( + mutable_cf_options_.compaction_options_universal.max_merge_width); + const std::size_t min_merge_width = static_cast( + mutable_cf_options_.compaction_options_universal.min_merge_width); + const uint64_t max_size_amplification_percent = + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent; + const uint64_t base_sr_size = sorted_runs_[end_index].size; + + // Leave at least 1 L0 file and 2 input sorted runs after exclusion + const std::size_t max_num_l0_to_exclude = + std::min(num_l0_input_pre_exclusion - 1, end_index - start_index - 1); + // In universal compaction, sorted runs from non L0 levels are counted + // toward `level0_stop_writes_trigger`. Therefore we need to subtract the + // total number of sorted runs picked originally for this compaction from + // `level0_stop_writes_trigger` to calculate + // `num_extra_l0_before_write_stop` + const std::size_t num_extra_l0_before_write_stop = + level0_stop_writes_trigger - + std::min(level0_stop_writes_trigger, end_index - start_index + 1); + const std::size_t num_l0_to_exclude_for_max_merge_width = + std::min(max_merge_width - + std::min(max_merge_width, num_extra_l0_before_write_stop), + max_num_l0_to_exclude); + const std::size_t num_l0_to_exclude_for_min_merge_width = + std::min(min_merge_width - + std::min(min_merge_width, num_extra_l0_before_write_stop), + max_num_l0_to_exclude); + + std::size_t num_l0_to_exclude = 0; + uint64_t candidate_size_post_exclusion = candidate_size; + + for (std::size_t possible_num_l0_to_exclude = + num_l0_to_exclude_for_min_merge_width; + possible_num_l0_to_exclude <= num_l0_to_exclude_for_max_merge_width; + ++possible_num_l0_to_exclude) { + uint64_t current_candidate_size = candidate_size_post_exclusion; + for (std::size_t j = num_l0_to_exclude; j < possible_num_l0_to_exclude; + ++j) { + current_candidate_size -= + sorted_runs_.at(start_index + j).compensated_file_size; + } + + // To ensure the compaction score before and after exclusion is similar + // so this exclusion will not make the size amp compaction of + // interest invalid from running as a size amp compaction as long as its + // pre-exclusion compaction score satisfies the condition to run. + if (current_candidate_size * 100 < + max_size_amplification_percent * base_sr_size || + current_candidate_size < candidate_size * 9 / 10) { + break; + } + num_l0_to_exclude = possible_num_l0_to_exclude; + candidate_size_post_exclusion = current_candidate_size; + } + + start_index += num_l0_to_exclude; + candidate_size = candidate_size_post_exclusion; + return num_l0_to_exclude; + } + const ImmutableOptions& ioptions_; const InternalKeyComparator* icmp_; double score_; @@ -778,85 +879,67 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns( // Look at overall size amplification. If size amplification // exceeds the configured value, then do a compaction -// of the candidate files all the way upto the earliest -// base file (overrides configured values of file-size ratios, -// min_merge_width and max_merge_width). -// +// on longest span of candidate files without conflict with other compactions +// ending at the earliest base file (overriding configured values of file-size +// ratios, min_merge_width and max_merge_width). Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() { - // percentage flexibility while reducing size amplification - uint64_t ratio = mutable_cf_options_.compaction_options_universal - .max_size_amplification_percent; - - unsigned int candidate_count = 0; - uint64_t candidate_size = 0; - size_t start_index = 0; - const SortedRun* sr = nullptr; - assert(!sorted_runs_.empty()); - if (sorted_runs_.back().being_compacted) { + + const size_t end_index = ShouldSkipLastSortedRunForSizeAmpCompaction() + ? sorted_runs_.size() - 2 + : sorted_runs_.size() - 1; + if (sorted_runs_[end_index].being_compacted) { return nullptr; } + const uint64_t base_sr_size = sorted_runs_[end_index].size; + size_t start_index = end_index; + uint64_t candidate_size = 0; + size_t num_l0_files = 0; - // Skip files that are already being compacted - for (size_t loop = 0; loop + 1 < sorted_runs_.size(); loop++) { - sr = &sorted_runs_[loop]; - if (!sr->being_compacted) { - start_index = loop; // Consider this as the first candidate. + // Get longest span (i.e, [start_index, end_index]) of available sorted runs + while (start_index > 0) { + const SortedRun* sr = &sorted_runs_[start_index - 1]; + if (sr->being_compacted) { + char file_num_buf[kFormatFileNumberBufSize]; + sr->Dump(file_num_buf, sizeof(file_num_buf), true); + ROCKS_LOG_BUFFER( + log_buffer_, + "[%s] Universal: stopping at sorted run undergoing compaction: " + "%s[%" ROCKSDB_PRIszt "]", + cf_name_.c_str(), file_num_buf, start_index - 1); break; } - char file_num_buf[kFormatFileNumberBufSize]; - sr->Dump(file_num_buf, sizeof(file_num_buf), true); - ROCKS_LOG_BUFFER(log_buffer_, - "[%s] Universal: skipping %s[%d] compacted %s", - cf_name_.c_str(), file_num_buf, loop, - " cannot be a candidate to reduce size amp.\n"); - sr = nullptr; + candidate_size += sr->compensated_file_size; + num_l0_files += sr->level == 0 ? 1 : 0; + --start_index; } - if (sr == nullptr) { - return nullptr; // no candidate files + if (start_index == end_index) { + return nullptr; } + + { + const size_t num_l0_to_exclude = MightExcludeNewL0sToReduceWriteStop( + num_l0_files, end_index, start_index, candidate_size); + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] Universal: Excluding %" ROCKSDB_PRIszt + " latest L0 files to reduce potential write stop " + "triggered by `level0_stop_writes_trigger`", + cf_name_.c_str(), num_l0_to_exclude); + } + { char file_num_buf[kFormatFileNumberBufSize]; - sr->Dump(file_num_buf, sizeof(file_num_buf), true); + sorted_runs_[start_index].Dump(file_num_buf, sizeof(file_num_buf), true); ROCKS_LOG_BUFFER( log_buffer_, "[%s] Universal: First candidate %s[%" ROCKSDB_PRIszt "] %s", cf_name_.c_str(), file_num_buf, start_index, " to reduce size amp.\n"); } - // size of the base sorted run for size amp calculation - uint64_t base_sr_size = sorted_runs_.back().size; - size_t sr_end_idx = sorted_runs_.size() - 1; - // If tiered compaction is enabled and the last sorted run is the last level - if (ioptions_.preclude_last_level_data_seconds > 0 && - ioptions_.num_levels > 2 && - sorted_runs_.back().level == ioptions_.num_levels - 1 && - sorted_runs_.size() > 1) { - sr_end_idx = sorted_runs_.size() - 2; - base_sr_size = sorted_runs_[sr_end_idx].size; - } - - // keep adding up all the remaining files - for (size_t loop = start_index; loop < sr_end_idx; loop++) { - sr = &sorted_runs_[loop]; - if (sr->being_compacted) { - // TODO with incremental compaction is supported, we might want to - // schedule some incremental compactions in parallel if needed. - char file_num_buf[kFormatFileNumberBufSize]; - sr->Dump(file_num_buf, sizeof(file_num_buf), true); - ROCKS_LOG_BUFFER( - log_buffer_, "[%s] Universal: Possible candidate %s[%d] %s", - cf_name_.c_str(), file_num_buf, start_index, - " is already being compacted. No size amp reduction possible.\n"); - return nullptr; - } - candidate_size += sr->compensated_file_size; - candidate_count++; - } - if (candidate_count == 0) { - return nullptr; - } + // percentage flexibility while reducing size amplification + const uint64_t ratio = mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent; // size amplification = percentage of additional size if (candidate_size * 100 < ratio * base_sr_size) { @@ -893,7 +976,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() { } } return PickCompactionWithSortedRunRange( - start_index, sr_end_idx, CompactionReason::kUniversalSizeAmplification); + start_index, end_index, CompactionReason::kUniversalSizeAmplification); } Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp( @@ -1442,4 +1525,3 @@ uint64_t UniversalCompactionBuilder::GetMaxOverlappingBytes() const { } } } // namespace ROCKSDB_NAMESPACE - diff --git a/db/compaction/tiered_compaction_test.cc b/db/compaction/tiered_compaction_test.cc index d8aa229df..779b980d8 100644 --- a/db/compaction/tiered_compaction_test.cc +++ b/db/compaction/tiered_compaction_test.cc @@ -1202,20 +1202,126 @@ TEST_P(TieredCompactionTest, RangeBasedTieredStorageLevel) { ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); - ASSERT_EQ( options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), 1); + + // Tests that we only compact keys up to penultimate level + // that are within penultimate level input's internal key range. + { + MutexLock l(&mutex); + hot_start = Key(0); + hot_end = Key(100); + } + const Snapshot* temp_snap = db_->GetSnapshot(); + // Key(0) and Key(1) here are inserted with higher sequence number + // than Key(0) and Key(1) inserted above. + // Only Key(0) in last level will be compacted up, not Key(1). + ASSERT_OK(Put(Key(0), "value" + std::to_string(0))); + ASSERT_OK(Put(Key(1), "value" + std::to_string(100))); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + { + std::vector metas; + db_->GetLiveFilesMetaData(&metas); + for (const auto& f : metas) { + if (f.temperature == Temperature::kUnknown) { + // Expect Key(0), Key(0), Key(1) + ASSERT_EQ(f.num_entries, 3); + ASSERT_EQ(f.smallestkey, Key(0)); + ASSERT_EQ(f.largestkey, Key(1)); + } else { + ASSERT_EQ(f.temperature, Temperature::kCold); + // Key(2)-Key(49) and Key(100). + ASSERT_EQ(f.num_entries, 50); + } + } + } + db_->ReleaseSnapshot(temp_snap); } INSTANTIATE_TEST_CASE_P(TieredCompactionTest, TieredCompactionTest, testing::Bool()); +TEST_P(TieredCompactionTest, CheckInternalKeyRange) { + // When compacting keys from the last level to penultimate level, + // output to penultimate level should be within internal key range + // of input files from penultimate level. + // Set up: + // L5: + // File 1: DeleteRange[1, 3)@4, File 2: [3@5, 100@6] + // L6: + // File 3: [2@1, 3@2], File 4: [50@3] + // + // When File 1 and File 3 are being compacted, + // Key(3) cannot be compacted up, otherwise it causes + // inconsistency where File 3's Key(3) has a lower sequence number + // than File 2's Key(3). + const int kNumLevels = 7; + auto options = CurrentOptions(); + SetColdTemperature(options); + options.level_compaction_dynamic_level_bytes = true; + options.num_levels = kNumLevels; + options.statistics = CreateDBStatistics(); + options.max_subcompactions = 10; + options.preclude_last_level_data_seconds = 10000; + DestroyAndReopen(options); + auto cmp = options.comparator; + + std::string hot_start = Key(0); + std::string hot_end = Key(0); + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + cmp->Compare(context->key, hot_start) >= 0 && + cmp->Compare(context->key, hot_end) < 0; + }); + SyncPoint::GetInstance()->EnableProcessing(); + // File 1 + ASSERT_OK(Put(Key(2), "val2")); + ASSERT_OK(Put(Key(3), "val3")); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + // File 2 + ASSERT_OK(Put(Key(50), "val50")); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + + const Snapshot* snapshot = db_->GetSnapshot(); + hot_end = Key(100); + std::string start = Key(1); + std::string end = Key(3); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + // File 3 + ASSERT_OK(Put(Key(3), "vall")); + ASSERT_OK(Put(Key(100), "val100")); + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + // Try to compact keys up + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + start = Key(1); + end = Key(2); + Slice begin_slice(start); + Slice end_slice(end); + ASSERT_OK(db_->CompactRange(cro, &begin_slice, &end_slice)); + // Without internal key range checking, we get the following error: + // Corruption: force_consistency_checks(DEBUG): VersionBuilder: L5 has + // overlapping ranges: file #18 largest key: '6B6579303030303033' seq:102, + // type:1 vs. file #15 smallest key: '6B6579303030303033' seq:104, type:1 + db_->ReleaseSnapshot(snapshot); +} + class PrecludeLastLevelTest : public DBTestBase { public: PrecludeLastLevelTest() : DBTestBase("preclude_last_level_test", /*env_do_fsync=*/false) { mock_clock_ = std::make_shared(env_->GetSystemClock()); + mock_clock_->SetCurrentTime(kMockStartTime); mock_env_ = std::make_unique(env_, mock_clock_); } @@ -1223,6 +1329,10 @@ class PrecludeLastLevelTest : public DBTestBase { std::unique_ptr mock_env_; std::shared_ptr mock_clock_; + // Sufficient starting time that preserve time doesn't under-flow into + // pre-history + static constexpr uint32_t kMockStartTime = 10000000; + void SetUp() override { mock_clock_->InstallTimedWaitFixCallback(); SyncPoint::GetInstance()->SetCallBack( @@ -1231,7 +1341,7 @@ class PrecludeLastLevelTest : public DBTestBase { reinterpret_cast(arg); periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get()); }); - mock_clock_->SetCurrentTime(0); + mock_clock_->SetCurrentTime(kMockStartTime); } }; @@ -1249,11 +1359,6 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) { options.num_levels = kNumLevels; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun( - [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); - int sst_num = 0; // Write files that are overlap and enough to trigger compaction for (; sst_num < kNumTrigger; sst_num++) { @@ -1311,11 +1416,6 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) { options.num_levels = kNumLevels; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun( - [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); - int sst_num = 0; // Write files that are overlap and enough to trigger compaction for (; sst_num < kNumTrigger; sst_num++) { @@ -1387,11 +1487,6 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) { options.num_levels = kNumLevels; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun( - [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); - int sst_num = 0; // Write files that are overlap and enough to trigger compaction for (; sst_num < kNumTrigger; sst_num++) { @@ -1514,11 +1609,6 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) { options.num_levels = kNumLevels; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun( - [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); - int sst_num = 0; // Write files that are overlap and enough to trigger compaction for (; sst_num < kNumTrigger; sst_num++) { @@ -1592,11 +1682,6 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) { options.num_levels = kNumLevels; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun( - [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); - Random rnd(301); int sst_num = 0; // Write files that are overlap and enough to trigger compaction @@ -1906,11 +1991,6 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) { options.num_levels = kNumLevels; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun( - [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); - Random rnd(301); for (int i = 0; i < 300; i++) { @@ -2017,12 +2097,6 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) { options.target_file_size_base = kFileBytes; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun([&] { - mock_clock_->MockSleepForSeconds(static_cast(kSecondsPerKey)); - }); - // Flush an L0 file with the following contents (new to old): // // Range deletions [4, 6) [7, 8) [9, 11) @@ -2139,7 +2213,6 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) { Close(); } - } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/comparator_db_test.cc b/db/comparator_db_test.cc index e5e3493b3..0bf79bef1 100644 --- a/db/comparator_db_test.cc +++ b/db/comparator_db_test.cc @@ -77,7 +77,7 @@ void DoRandomIteraratorTest(DB* db, std::vector source_strings, for (int i = 0; i < num_writes; i++) { if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) { - db->Flush(FlushOptions()); + ASSERT_OK(db->Flush(FlushOptions())); } int type = rnd->Uniform(2); @@ -156,6 +156,7 @@ void DoRandomIteraratorTest(DB* db, std::vector source_strings, if (map.find(key) == map.end()) { ASSERT_TRUE(status.IsNotFound()); } else { + ASSERT_OK(status); ASSERT_EQ(map[key], result); } break; @@ -164,6 +165,7 @@ void DoRandomIteraratorTest(DB* db, std::vector source_strings, AssertItersEqual(iter.get(), result_iter.get()); is_valid = iter->Valid(); } + ASSERT_OK(iter->status()); } class DoubleComparator : public Comparator { diff --git a/db/convenience.cc b/db/convenience.cc index 32cdfafaa..08bddc8e8 100644 --- a/db/convenience.cc +++ b/db/convenience.cc @@ -7,6 +7,7 @@ #include "rocksdb/convenience.h" +#include "db/convenience_impl.h" #include "db/db_impl/db_impl.h" #include "util/cast_util.h" @@ -39,9 +40,25 @@ Status VerifySstFileChecksum(const Options& options, } Status VerifySstFileChecksum(const Options& options, const EnvOptions& env_options, - const ReadOptions& read_options, + const ReadOptions& _read_options, const std::string& file_path, const SequenceNumber& largest_seqno) { + if (_read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Can only call VerifySstFileChecksum with `ReadOptions::io_activity` " + "is " + "`Env::IOActivity::kUnknown`"); + } + ReadOptions read_options(_read_options); + return VerifySstFileChecksumInternal(options, env_options, read_options, + file_path, largest_seqno); +} + +Status VerifySstFileChecksumInternal(const Options& options, + const EnvOptions& env_options, + const ReadOptions& read_options, + const std::string& file_path, + const SequenceNumber& largest_seqno) { std::unique_ptr file; uint64_t file_size; InternalKeyComparator internal_comparator(options.comparator); @@ -68,8 +85,8 @@ Status VerifySstFileChecksum(const Options& options, !kImmortal, false /* force_direct_prefetch */, -1 /* level */); reader_options.largest_seqno = largest_seqno; s = ioptions.table_factory->NewTableReader( - reader_options, std::move(file_reader), file_size, &table_reader, - false /* prefetch_index_and_filter_in_cache */); + read_options, reader_options, std::move(file_reader), file_size, + &table_reader, false /* prefetch_index_and_filter_in_cache */); if (!s.ok()) { return s; } diff --git a/db/convenience_impl.h b/db/convenience_impl.h new file mode 100644 index 000000000..32f4476bd --- /dev/null +++ b/db/convenience_impl.h @@ -0,0 +1,15 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include "rocksdb/db.h" + +namespace ROCKSDB_NAMESPACE { +Status VerifySstFileChecksumInternal(const Options& options, + const EnvOptions& env_options, + const ReadOptions& read_options, + const std::string& file_path, + const SequenceNumber& largest_seqno = 0); +} // namespace ROCKSDB_NAMESPACE diff --git a/db/db_basic_test.cc b/db/db_basic_test.cc index 063b99839..0b0383ae0 100644 --- a/db/db_basic_test.cc +++ b/db/db_basic_test.cc @@ -138,6 +138,7 @@ TEST_F(DBBasicTest, ReadOnlyDB) { ASSERT_OK(iter->status()); ++count; } + ASSERT_OK(iter->status()); // Always expect two keys: "foo" and "bar" ASSERT_EQ(count, 2); }; @@ -1204,9 +1205,23 @@ TEST_F(DBBasicTest, DBClose) { delete db; ASSERT_EQ(env->GetCloseCount(), 2); + // close by WaitForCompact() with close_db option + options.create_if_missing = false; + s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + WaitForCompactOptions wait_for_compact_options = WaitForCompactOptions(); + wait_for_compact_options.close_db = true; + s = db->WaitForCompact(wait_for_compact_options); + ASSERT_EQ(env->GetCloseCount(), 3); + // see TestLogger::CloseHelper() + ASSERT_EQ(s, Status::IOError()); + + delete db; + ASSERT_EQ(env->GetCloseCount(), 3); + // Provide our own logger and ensure DB::Close() does not close it options.info_log.reset(new TestEnv::TestLogger(env)); - options.create_if_missing = false; s = DB::Open(options, dbname, &db); ASSERT_OK(s); ASSERT_TRUE(db != nullptr); @@ -1214,9 +1229,9 @@ TEST_F(DBBasicTest, DBClose) { s = db->Close(); ASSERT_EQ(s, Status::OK()); delete db; - ASSERT_EQ(env->GetCloseCount(), 2); - options.info_log.reset(); ASSERT_EQ(env->GetCloseCount(), 3); + options.info_log.reset(); + ASSERT_EQ(env->GetCloseCount(), 4); } TEST_F(DBBasicTest, DBCloseAllDirectoryFDs) { @@ -1413,10 +1428,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) { int retries = 0; bool last_try = false; ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( - "DBImpl::MultiGet::LastTry", [&](void* /*arg*/) { - last_try = true; - ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); - }); + "DBImpl::MultiGet::LastTry", [&](void* /*arg*/) { last_try = true; }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) { if (last_try) { @@ -1433,8 +1445,28 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) { } } }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::MultiGet::AfterLastTryRefSV", + "DBMultiGetTestWithParam::MultiGetMultiCFMutex:BeforeCreateSV"}, + {"DBMultiGetTestWithParam::MultiGetMultiCFMutex:AfterCreateSV", + "DBImpl::MultiGet::BeforeLastTryUnRefSV"}, + }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + port::Thread create_sv_thread([this]() { + TEST_SYNC_POINT( + "DBMultiGetTestWithParam::MultiGetMultiCFMutex:BeforeCreateSV"); + // Create a new SuperVersion for each column family after last_try + // of MultiGet ref SuperVersion and before unref it. + for (int i = 0; i < 8; ++i) { + ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key", + "cf" + std::to_string(i) + "_val_after_last_try")); + ASSERT_OK(Flush(i)); + } + TEST_SYNC_POINT( + "DBMultiGetTestWithParam::MultiGetMultiCFMutex:AfterCreateSV"); + }); + std::vector cfs; std::vector keys; std::vector values; @@ -1446,6 +1478,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) { values = MultiGet(cfs, keys, nullptr, std::get<0>(GetParam()), std::get<1>(GetParam())); + create_sv_thread.join(); ASSERT_TRUE(last_try); ASSERT_EQ(values.size(), 8); for (unsigned int j = 0; j < values.size(); ++j) { @@ -1459,6 +1492,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) { ->cfd(); ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse); } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFSnapshot) { @@ -3621,10 +3655,12 @@ class DBBasicTestMultiGet : public DBTestBase { Status Insert(const Slice& key, Cache::ObjectPtr value, const CacheItemHelper* helper, size_t charge, - Handle** handle = nullptr, - Priority priority = Priority::LOW) override { + Handle** handle = nullptr, Priority priority = Priority::LOW, + const Slice& compressed = Slice(), + CompressionType type = kNoCompression) override { num_inserts_++; - return target_->Insert(key, value, helper, charge, handle, priority); + return target_->Insert(key, value, helper, charge, handle, priority, + compressed, type); } Handle* Lookup(const Slice& key, const CacheItemHelper* helper, @@ -4379,6 +4415,8 @@ TEST_F(DBBasicTest, ManifestWriteFailure) { options.create_if_missing = true; options.disable_auto_compactions = true; options.env = env_; + options.enable_blob_files = true; + options.blob_file_size = 0; DestroyAndReopen(options); ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Flush()); @@ -4399,6 +4437,11 @@ TEST_F(DBBasicTest, ManifestWriteFailure) { SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->EnableProcessing(); Reopen(options); + // The IO error was a mocked one from the `AfterSyncManifest` callback. The + // Flush's VersionEdit actually made it into the Manifest. So these keys can + // be read back. Read them to check all live sst files and blob files. + ASSERT_EQ("bar", Get("foo")); + ASSERT_EQ("value", Get("key")); } TEST_F(DBBasicTest, DestroyDefaultCfHandle) { @@ -4436,7 +4479,7 @@ TEST_F(DBBasicTest, FailOpenIfLoggerCreationFail) { SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->SetCallBack( - "rocksdb::CreateLoggerFromOptions:AfterGetPath", [&](void* arg) { + "forstdb::CreateLoggerFromOptions:AfterGetPath", [&](void* arg) { auto* s = reinterpret_cast(arg); assert(s); *s = Status::IOError("Injected"); diff --git a/db/db_block_cache_test.cc b/db/db_block_cache_test.cc index 83a027f5e..938ca911c 100644 --- a/db/db_block_cache_test.cc +++ b/db/db_block_cache_test.cc @@ -294,7 +294,9 @@ class ReadOnlyCacheWrapper : public CacheWrapper { Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/, const CacheItemHelper* /*helper*/, size_t /*charge*/, - Handle** /*handle*/, Priority /*priority*/) override { + Handle** /*handle*/, Priority /*priority*/, + const Slice& /*compressed*/, + CompressionType /*type*/) override { return Status::NotSupported(); } }; @@ -387,6 +389,7 @@ TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) { while (iter->Valid()) { iter->Next(); } + ASSERT_OK(iter->status()); delete iter; iter = nullptr; } @@ -628,13 +631,15 @@ class MockCache : public LRUCache { Status Insert(const Slice& key, Cache::ObjectPtr value, const Cache::CacheItemHelper* helper, size_t charge, - Handle** handle, Priority priority) override { + Handle** handle, Priority priority, const Slice& compressed, + CompressionType type) override { if (priority == Priority::LOW) { low_pri_insert_count++; } else { high_pri_insert_count++; } - return LRUCache::Insert(key, value, helper, charge, handle, priority); + return LRUCache::Insert(key, value, helper, charge, handle, priority, + compressed, type); } }; @@ -741,10 +746,15 @@ TEST_F(DBBlockCacheTest, AddRedundantStats) { int iterations_tested = 0; for (std::shared_ptr base_cache : {NewLRUCache(capacity, num_shard_bits), + // FixedHyperClockCache HyperClockCacheOptions( capacity, BlockBasedTableOptions().block_size /*estimated_value_size*/, num_shard_bits) + .MakeSharedCache(), + // AutoHyperClockCache + HyperClockCacheOptions(capacity, 0 /*estimated_value_size*/, + num_shard_bits) .MakeSharedCache()}) { if (!base_cache) { // Skip clock cache when not supported diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index 69face563..9bd5c11b6 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -654,7 +654,7 @@ TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) { for (i = 0; i < maxKey; i++) { ASSERT_OK(Put(Key(i), Key(i))); } - Flush(); + ASSERT_OK(Flush()); }; auto GetFn = [&]() { int i; @@ -792,7 +792,7 @@ TEST_F(DBBloomFilterTest, BloomFilterRate) { } // Add a large key to make the file contain wide range ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555))); - Flush(1); + ASSERT_OK(Flush(1)); // Check if they can be found for (int i = 0; i < maxKey; i++) { @@ -1696,7 +1696,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) { table_options.format_version = 5; options.table_factory.reset(NewBlockBasedTableFactory(table_options)); - TryReopen(options); + ASSERT_OK(TryReopen(options)); CreateAndReopenWithCF({fifo ? "abe" : "bob"}, options); const int maxKey = 10000; @@ -1705,7 +1705,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) { } // Add a large key to make the file contain wide range ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555))); - Flush(1); + ASSERT_OK(Flush(1)); EXPECT_EQ(policy->DumpTestReport(), fifo ? "cf=abe,s=kCompactionStyleFIFO,n=7,l=0,b=0,r=kFlush\n" : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n"); @@ -1713,7 +1713,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) { for (int i = maxKey / 2; i < maxKey; i++) { ASSERT_OK(Put(1, Key(i), Key(i))); } - Flush(1); + ASSERT_OK(Flush(1)); EXPECT_EQ(policy->DumpTestReport(), fifo ? "cf=abe,s=kCompactionStyleFIFO,n=7,l=0,b=0,r=kFlush\n" : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n"); @@ -1775,6 +1775,64 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) { } } +TEST_F(DBBloomFilterTest, MutatingRibbonFilterPolicy) { + // Test that RibbonFilterPolicy has a mutable bloom_before_level fields that + // can be updated through SetOptions + + Options options = CurrentOptions(); + options.statistics = CreateDBStatistics(); + auto& stats = *options.statistics; + BlockBasedTableOptions table_options; + // First config forces Bloom filter, to establish a baseline before + // SetOptions(). + table_options.filter_policy.reset(NewRibbonFilterPolicy(10, INT_MAX)); + double expected_bpk = 10.0; + // Other configs to try, with approx expected bits per key + std::vector> configs = {{"-1", 7.0}, + {"0", 10.0}}; + + table_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + ASSERT_OK(TryReopen(options)); + + char v[] = "a"; + + for (;; ++(v[0])) { + const int maxKey = 8000; + for (int i = 0; i < maxKey; i++) { + ASSERT_OK(Put(Key(i), v)); + } + ASSERT_OK(Flush()); + + for (int i = 0; i < maxKey; i++) { + ASSERT_EQ(Get(Key(i)), v); + } + + uint64_t filter_bytes = + stats.getAndResetTickerCount(BLOCK_CACHE_FILTER_BYTES_INSERT); + + EXPECT_NEAR(filter_bytes * 8.0 / maxKey, expected_bpk, 0.3); + + if (configs.empty()) { + break; + } + + ASSERT_OK( + db_->SetOptions({{"table_factory.filter_policy.bloom_before_level", + configs.back().first}})); + + // Ensure original object is mutated + std::string val; + ASSERT_OK( + table_options.filter_policy->GetOption({}, "bloom_before_level", &val)); + ASSERT_EQ(configs.back().first, val); + + expected_bpk = configs.back().second; + configs.pop_back(); + } +} + class SliceTransformLimitedDomain : public SliceTransform { const char* Name() const override { return "SliceTransformLimitedDomain"; } @@ -1847,6 +1905,7 @@ TEST_F(DBBloomFilterTest, PrefixExtractorWithFilter2) { for (iter->Seek("zzzzz_AAAA"); iter->Valid(); iter->Next()) { iter_res.emplace_back(iter->value().ToString()); } + ASSERT_OK(iter->status()); std::vector expected_res = {"val1", "val2", "val3", "val4"}; ASSERT_EQ(iter_res, expected_res); @@ -2261,7 +2320,7 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTest) { ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count); ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count); - Flush(); + ASSERT_OK(Flush()); // sanity checks ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count); @@ -2311,7 +2370,7 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) { ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count); ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count); - Flush(); + ASSERT_OK(Flush()); iter.reset(dbfull()->NewIterator(ReadOptions())); @@ -2379,7 +2438,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) { snprintf(buf, sizeof(buf), "%02d______:end", i + 1); keystr = std::string(buf); ASSERT_OK(dbtest->Put(keystr, keystr)); - dbtest->Flush(); + ASSERT_OK(dbtest->Flush()); } // GROUP 2 @@ -2390,7 +2449,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) { snprintf(buf, sizeof(buf), "%02d______:end", small_range_sstfiles + i + 1); keystr = std::string(buf); ASSERT_OK(dbtest->Put(keystr, keystr)); - dbtest->Flush(); + ASSERT_OK(dbtest->Flush()); } } } // anonymous namespace @@ -2853,7 +2912,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) { ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Put("foq1", "bar1")); ASSERT_OK(Put("fpa", "0")); - dbfull()->Flush(FlushOptions()); + ASSERT_OK(dbfull()->Flush(FlushOptions())); std::unique_ptr iter_old(db_->NewIterator(read_options)); ASSERT_EQ(CountIter(iter_old, "foo"), 4); EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); @@ -2981,7 +3040,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) { ASSERT_OK(Put(2, "foo5", "bar5")); ASSERT_OK(Put(2, "foq6", "bar6")); ASSERT_OK(Put(2, "fpq7", "bar7")); - dbfull()->Flush(FlushOptions()); + ASSERT_OK(dbfull()->Flush(FlushOptions())); { std::unique_ptr iter( db_->NewIterator(read_options, handles_[2])); @@ -3031,17 +3090,17 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) { ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Put("foo1", "bar1")); ASSERT_OK(Put("fpa", "0")); - dbfull()->Flush(FlushOptions()); + ASSERT_OK(dbfull()->Flush(FlushOptions())); ASSERT_OK(Put("foo3", "bar3")); ASSERT_OK(Put("foo4", "bar4")); ASSERT_OK(Put("foo5", "bar5")); ASSERT_OK(Put("fpb", "1")); - dbfull()->Flush(FlushOptions()); + ASSERT_OK(dbfull()->Flush(FlushOptions())); ASSERT_OK(Put("foo6", "bar6")); ASSERT_OK(Put("foo7", "bar7")); ASSERT_OK(Put("foo8", "bar8")); ASSERT_OK(Put("fpc", "2")); - dbfull()->Flush(FlushOptions()); + ASSERT_OK(dbfull()->Flush(FlushOptions())); ReadOptions read_options; read_options.prefix_same_as_start = true; diff --git a/db/db_compaction_filter_test.cc b/db/db_compaction_filter_test.cc index 596dfefc5..44c406c49 100644 --- a/db/db_compaction_filter_test.cc +++ b/db/db_compaction_filter_test.cc @@ -166,9 +166,12 @@ class ChangeFilter : public CompactionFilter { class KeepFilterFactory : public CompactionFilterFactory { public: explicit KeepFilterFactory(bool check_context = false, - bool check_context_cf_id = false) + bool check_context_cf_id = false, + bool check_context_input_table_properties = false) : check_context_(check_context), check_context_cf_id_(check_context_cf_id), + check_context_input_table_properties_( + check_context_input_table_properties), compaction_filter_created_(false) {} std::unique_ptr CreateCompactionFilter( @@ -176,6 +179,11 @@ class KeepFilterFactory : public CompactionFilterFactory { if (check_context_) { EXPECT_EQ(expect_full_compaction_.load(), context.is_full_compaction); EXPECT_EQ(expect_manual_compaction_.load(), context.is_manual_compaction); + EXPECT_EQ(expect_input_start_level_.load(), context.input_start_level); + } + if (check_context_input_table_properties_) { + EXPECT_TRUE(expect_input_table_properties_ == + context.input_table_properties); } if (check_context_cf_id_) { EXPECT_EQ(expect_cf_id_.load(), context.column_family_id); @@ -189,9 +197,15 @@ class KeepFilterFactory : public CompactionFilterFactory { const char* Name() const override { return "KeepFilterFactory"; } bool check_context_; bool check_context_cf_id_; + // `check_context_input_table_properties_` can be true only when access to + // `expect_input_table_properties_` is syncronized since we can't have + // std::atomic unfortunately + bool check_context_input_table_properties_; std::atomic_bool expect_full_compaction_; std::atomic_bool expect_manual_compaction_; std::atomic expect_cf_id_; + std::atomic expect_input_start_level_; + TablePropertiesCollection expect_input_table_properties_; bool compaction_filter_created_; }; @@ -654,7 +668,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) { } TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) { - KeepFilterFactory* filter = new KeepFilterFactory(true, true); + KeepFilterFactory* filter = new KeepFilterFactory( + true /* check_context */, true /* check_context_cf_id */, + true /* check_context_input_table_properties */); Options options = CurrentOptions(); options.compaction_style = kCompactionStyleUniversal; @@ -662,8 +678,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) { options.compression = kNoCompression; options.level0_file_num_compaction_trigger = 8; Reopen(options); + const int kNumFiles = 3; int num_keys_per_file = 400; - for (int j = 0; j < 3; j++) { + for (int j = 0; j < kNumFiles; j++) { // Write several keys. const std::string value(10, 'x'); for (int i = 0; i < num_keys_per_file; i++) { @@ -683,6 +700,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) { filter->expect_manual_compaction_.store(true); filter->expect_full_compaction_.store(true); filter->expect_cf_id_.store(0); + filter->expect_input_start_level_.store(0); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables( + &filter->expect_input_table_properties_)); + ASSERT_TRUE(filter->expect_input_table_properties_.size() == kNumFiles); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_EQ(cfilter_count, 700); ASSERT_EQ(NumSortedRuns(0), 1); @@ -783,6 +805,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) { count++; iter->Next(); } + ASSERT_OK(iter->status()); ASSERT_EQ(count, 6); read_options.snapshot = nullptr; std::unique_ptr iter1(db_->NewIterator(read_options)); @@ -793,6 +816,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) { count++; iter1->Next(); } + ASSERT_OK(iter1->status()); // We have deleted 10 keys from 40 using the compaction filter // Keys 6-9 before the snapshot and 100-105 after the snapshot ASSERT_EQ(count, 30); diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index a1b4035e0..be88e198f 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -19,7 +19,7 @@ #include "rocksdb/concurrent_task_limiter.h" #include "rocksdb/experimental.h" #include "rocksdb/sst_file_writer.h" -#include "rocksdb/utilities/convenience.h" +#include "test_util/mock_time_env.h" #include "test_util/sync_point.h" #include "test_util/testutil.h" #include "util/concurrent_task_limiter_impl.h" @@ -76,7 +76,7 @@ class CompactionStatsCollector : public EventListener { class DBCompactionTest : public DBTestBase { public: DBCompactionTest() - : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) {} + : DBTestBase("db_compaction_test", /*env_do_fsync=*/false) {} protected: /* @@ -121,7 +121,7 @@ class DBCompactionTestWithParam public testing::WithParamInterface> { public: DBCompactionTestWithParam() - : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) { + : DBTestBase("db_compaction_test", /*env_do_fsync=*/false) { max_subcompactions_ = std::get<0>(GetParam()); exclusive_manual_compaction_ = std::get<1>(GetParam()); } @@ -140,7 +140,7 @@ class DBCompactionTestWithBottommostParam std::tuple> { public: DBCompactionTestWithBottommostParam() - : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) { + : DBTestBase("db_compaction_test", /*env_do_fsync=*/false) { bottommost_level_compaction_ = std::get<0>(GetParam()); } @@ -153,17 +153,23 @@ class DBCompactionDirectIOTest : public DBCompactionTest, DBCompactionDirectIOTest() : DBCompactionTest() {} }; +// Params: See WaitForCompactOptions for details class DBCompactionWaitForCompactTest : public DBTestBase, - public testing::WithParamInterface> { + public testing::WithParamInterface< + std::tuple> { public: DBCompactionWaitForCompactTest() - : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) { + : DBTestBase("db_compaction_test", /*env_do_fsync=*/false) { abort_on_pause_ = std::get<0>(GetParam()); flush_ = std::get<1>(GetParam()); + close_db_ = std::get<2>(GetParam()); + timeout_ = std::get<3>(GetParam()); } bool abort_on_pause_; bool flush_; + bool close_db_; + std::chrono::microseconds timeout_; Options options_; WaitForCompactOptions wait_for_compact_options_; @@ -179,6 +185,8 @@ class DBCompactionWaitForCompactTest wait_for_compact_options_ = WaitForCompactOptions(); wait_for_compact_options_.abort_on_pause = abort_on_pause_; wait_for_compact_options_.flush = flush_; + wait_for_compact_options_.close_db = close_db_; + wait_for_compact_options_.timeout = timeout_; DestroyAndReopen(options_); @@ -837,6 +845,20 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) { options.memtable_factory.reset( test::NewSpecialSkipListFactory(kNumKeysPerFile)); + CreateAndReopenWithCF({"one", "two", "three"}, options); + + Random rnd(301); + for (int cf = 0; cf < 4; cf++) { + // Make a trivial L1 for L0 to compact into. L2 will be large so debt ratio + // will not cause compaction pressure. + ASSERT_OK(Put(cf, Key(0), rnd.RandomString(102400))); + ASSERT_OK(Flush(cf)); + MoveFilesToLevel(2, cf); + ASSERT_OK(Put(cf, Key(0), "")); + ASSERT_OK(Flush(cf)); + MoveFilesToLevel(1, cf); + } + // Block all threads in thread pool. const size_t kTotalTasks = 4; env_->SetBackgroundThreads(4, Env::LOW); @@ -847,9 +869,6 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) { sleeping_tasks[i].WaitUntilSleeping(); } - CreateAndReopenWithCF({"one", "two", "three"}, options); - - Random rnd(301); for (int cf = 0; cf < 4; cf++) { for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { for (int i = 0; i < kNumKeysPerFile; i++) { @@ -2665,6 +2684,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) { keys_in_db.append(iter->key().ToString()); keys_in_db.push_back(','); } + ASSERT_OK(iter->status()); delete iter; std::string expected_keys; @@ -3331,12 +3351,19 @@ TEST_F(DBCompactionTest, SuggestCompactRangeNoTwoLevel0Compactions) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } -INSTANTIATE_TEST_CASE_P(DBCompactionWaitForCompactTest, - DBCompactionWaitForCompactTest, - ::testing::Values(std::make_tuple(false, false), - std::make_tuple(false, true), - std::make_tuple(true, false), - std::make_tuple(true, true))); +INSTANTIATE_TEST_CASE_P( + DBCompactionWaitForCompactTest, DBCompactionWaitForCompactTest, + ::testing::Combine( + testing::Bool() /* abort_on_pause */, testing::Bool() /* flush */, + testing::Bool() /* close_db */, + testing::Values( + std::chrono::microseconds::zero(), + std::chrono::microseconds{ + 60 * 60 * + 1000000ULL} /* timeout */))); // 1 hour (long enough to + // make sure that tests + // don't fail unexpectedly + // when running slow) TEST_P(DBCompactionWaitForCompactTest, WaitForCompactWaitsOnCompactionToFinish) { @@ -3476,19 +3503,19 @@ TEST_P(DBCompactionWaitForCompactTest, WaitForCompactWithOptionToFlush) { ASSERT_EQ("2", FilesPerLevel()); ASSERT_OK(dbfull()->WaitForCompact(wait_for_compact_options_)); - if (flush_) { - ASSERT_EQ("1,2", FilesPerLevel()); - ASSERT_EQ(1, compaction_finished); - ASSERT_EQ(1, flush_finished); - } else { - ASSERT_EQ(0, compaction_finished); - ASSERT_EQ(0, flush_finished); - ASSERT_EQ("2", FilesPerLevel()); + ASSERT_EQ(flush_, compaction_finished); + ASSERT_EQ(flush_, flush_finished); + + if (!close_db_) { + std::string expected_files_per_level = flush_ ? "1,2" : "2"; + ASSERT_EQ(expected_files_per_level, FilesPerLevel()); } compaction_finished = 0; flush_finished = 0; - Close(); + if (!close_db_) { + Close(); + } Reopen(options_); ASSERT_EQ(0, flush_finished); @@ -3503,12 +3530,123 @@ TEST_P(DBCompactionWaitForCompactTest, WaitForCompactWithOptionToFlush) { ASSERT_EQ(1, compaction_finished); } - ASSERT_EQ("1,2", FilesPerLevel()); + if (!close_db_) { + ASSERT_EQ("1,2", FilesPerLevel()); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(DBCompactionWaitForCompactTest, + WaitForCompactWithOptionToFlushAndCloseDB) { + // After creating enough L0 files that one more file will trigger the + // compaction, write some data in memtable (WAL disabled). Calls + // WaitForCompact. If flush option is true, WaitForCompact will flush the + // memtable to a new L0 file which will trigger compaction. We expect the + // no-op second flush upon closing because WAL is disabled + // (has_unpersisted_data_ true) Check to make sure there's no extra L0 file + // created from WAL. Re-opening DB won't trigger any flush or compaction + + std::atomic_int compaction_finished = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:Finish", + [&](void*) { compaction_finished++; }); + + std::atomic_int flush_finished = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FlushJob::End", [&](void*) { flush_finished++; }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_FALSE(options_.avoid_flush_during_shutdown); + + // write to memtable, but no flush is needed at this point. + WriteOptions write_without_wal; + write_without_wal.disableWAL = true; + ASSERT_OK(Put(Key(0), "some random string", write_without_wal)); + ASSERT_EQ(0, compaction_finished); + ASSERT_EQ(0, flush_finished); + ASSERT_EQ("2", FilesPerLevel()); + + ASSERT_OK(dbfull()->WaitForCompact(wait_for_compact_options_)); + + int expected_flush_count = flush_ || close_db_; + ASSERT_EQ(expected_flush_count, flush_finished); + + if (!close_db_) { + // During CancelAllBackgroundWork(), a flush can be initiated due to + // unpersisted data (data that's still in the memtable when WAL is off). + // This results in an additional L0 file which can trigger a compaction. + // However, the compaction may not complete if the background thread's + // execution is slow enough for the front thread to set the 'shutting_down_' + // flag to true before the compaction job even starts. + ASSERT_EQ(expected_flush_count, compaction_finished); + Close(); + } + + // Because we had has_unpersisted_data_ = true, flush must have been triggered + // upon closing regardless of WaitForCompact. Reopen should have no flush + // debt. + flush_finished = 0; + Reopen(options_); + ASSERT_EQ(0, flush_finished); + + // However, if db was closed directly by calling Close(), instead + // of WaitForCompact with close_db option or we are in the scenario commented + // above, it's possible that the last compaction triggered by flushing + // unpersisted data was cancelled. Call WaitForCompact() here again to finish + // the compaction + if (compaction_finished == 0) { + ASSERT_OK(dbfull()->WaitForCompact(wait_for_compact_options_)); + } + ASSERT_EQ(1, compaction_finished); + if (!close_db_) { + ASSERT_EQ("1,2", FilesPerLevel()); + } ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); } +TEST_P(DBCompactionWaitForCompactTest, WaitForCompactToTimeout) { + // When timeout is set, this test makes CompactionJob hangs forever + // using sync point. This test also sets the timeout to be 1 ms for + // WaitForCompact to time out early. WaitForCompact() is expected to return + // Status::TimedOut. + // When timeout is not set, we expect WaitForCompact() to wait indefinitely. + // We don't want the test to hang forever. When timeout = 0, this test is not + // much different from WaitForCompactWaitsOnCompactionToFinish + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBCompactionTest::WaitForCompactToTimeout", + "CompactionJob::Run():Start"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Now trigger L0 compaction by adding a file + Random rnd(123); + GenerateNewRandomFile(&rnd, /* nowait */ true); + ASSERT_OK(Flush()); + + if (wait_for_compact_options_.timeout.count()) { + // Make timeout shorter to finish test early + wait_for_compact_options_.timeout = std::chrono::microseconds{1000}; + } else { + // if timeout is not set, WaitForCompact() will wait forever. We don't + // want test to hang forever. Just let compaction go through + TEST_SYNC_POINT("DBCompactionTest::WaitForCompactToTimeout"); + } + Status s = dbfull()->WaitForCompact(wait_for_compact_options_); + if (wait_for_compact_options_.timeout.count()) { + ASSERT_NOK(s); + ASSERT_TRUE(s.IsTimedOut()); + } else { + ASSERT_OK(s); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + static std::string ShortKey(int i) { assert(i < 10000); char buf[100]; @@ -4052,11 +4190,6 @@ TEST_F(DBCompactionTest, CompactBottomLevelFilesWithDeletions) { // files does not need to be preserved in case of a future snapshot. ASSERT_OK(Put(Key(0), "val")); ASSERT_NE(kMaxSequenceNumber, dbfull()->bottommost_files_mark_threshold_); - // release snapshot and wait for compactions to finish. Single-file - // compactions should be triggered, which reduce the size of each bottom-level - // file without changing file count. - db_->ReleaseSnapshot(snapshot); - ASSERT_EQ(kMaxSequenceNumber, dbfull()->bottommost_files_mark_threshold_); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { Compaction* compaction = reinterpret_cast(arg); @@ -4064,6 +4197,11 @@ TEST_F(DBCompactionTest, CompactBottomLevelFilesWithDeletions) { CompactionReason::kBottommostFiles); }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + // release snapshot and wait for compactions to finish. Single-file + // compactions should be triggered, which reduce the size of each bottom-level + // file without changing file count. + db_->ReleaseSnapshot(snapshot); + ASSERT_EQ(kMaxSequenceNumber, dbfull()->bottommost_files_mark_threshold_); ASSERT_OK(dbfull()->TEST_WaitForCompact()); db_->GetLiveFilesMetaData(&post_release_metadata); ASSERT_EQ(pre_release_metadata.size(), post_release_metadata.size()); @@ -4080,6 +4218,78 @@ TEST_F(DBCompactionTest, CompactBottomLevelFilesWithDeletions) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } +TEST_F(DBCompactionTest, DelayCompactBottomLevelFilesWithDeletions) { + // bottom-level files may contain deletions due to snapshots protecting the + // deleted keys. Once the snapshot is released and the files are old enough, + // we should see them undergo single-file compactions. + Options options = CurrentOptions(); + env_->SetMockSleep(); + options.bottommost_file_compaction_delay = 3600; + DestroyAndReopen(options); + CreateColumnFamilies({"one"}, options); + const int kNumKey = 100; + const int kValLen = 100; + + Random rnd(301); + for (int i = 0; i < kNumKey; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValLen))); + } + const Snapshot* snapshot = db_->GetSnapshot(); + for (int i = 0; i < kNumKey; i += 2) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + std::vector pre_release_metadata; + db_->GetLiveFilesMetaData(&pre_release_metadata); + ASSERT_EQ(1, pre_release_metadata.size()); + std::atomic_int compaction_count = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(compaction->compaction_reason() == + CompactionReason::kBottommostFiles); + compaction_count++; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + // just need to bump seqnum so ReleaseSnapshot knows the newest key in the SST + // files does not need to be preserved in case of a future snapshot. + ASSERT_OK(Put(Key(0), "val")); + ASSERT_NE(kMaxSequenceNumber, dbfull()->bottommost_files_mark_threshold_); + // release snapshot will not trigger compaction. + db_->ReleaseSnapshot(snapshot); + ASSERT_EQ(kMaxSequenceNumber, dbfull()->bottommost_files_mark_threshold_); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, compaction_count); + // Now the file is old enough for compaction. + env_->MockSleepForSeconds(3600); + // Another flush will trigger re-computation of the compaction score + // to find out that the file is qualified for compaction. + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(1, compaction_count); + + std::vector post_release_metadata; + db_->GetLiveFilesMetaData(&post_release_metadata); + ASSERT_EQ(2, post_release_metadata.size()); + + const auto& pre_file = pre_release_metadata[0]; + // Get the L1 (bottommost level) file. + const auto& post_file = post_release_metadata[0].level == 0 + ? post_release_metadata[1] + : post_release_metadata[0]; + + ASSERT_EQ(1, pre_file.level); + ASSERT_EQ(1, post_file.level); + // the file is smaller than it was before as it was rewritten without + // deletion markers/deleted keys. + ASSERT_LT(post_file.size, pre_file.size); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + TEST_F(DBCompactionTest, NoCompactBottomLevelFilesWithDeletions) { // bottom-level files may contain deletions due to snapshots protecting the // deleted keys. Once the snapshot is released, we should see files with many @@ -4595,9 +4805,9 @@ TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "VersionEdit::EncodeTo:VarintOldestAncesterTime", [&](void* arg) { if (if_restart && if_open_all_files) { - std::string* encoded_fieled = static_cast(arg); - *encoded_fieled = ""; - PutVarint64(encoded_fieled, 0); + std::string* encoded_field = static_cast(arg); + *encoded_field = ""; + PutVarint64(encoded_field, 0); } }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); @@ -4743,9 +4953,9 @@ TEST_F(DBCompactionTest, LevelPeriodicCompaction) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( "VersionEdit::EncodeTo:VarintFileCreationTime", [&](void* arg) { if (if_restart && if_open_all_files) { - std::string* encoded_fieled = static_cast(arg); - *encoded_fieled = ""; - PutVarint64(encoded_fieled, 0); + std::string* encoded_field = static_cast(arg); + *encoded_field = ""; + PutVarint64(encoded_field, 0); } }); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); @@ -4821,6 +5031,106 @@ TEST_F(DBCompactionTest, LevelPeriodicCompaction) { } } +TEST_F(DBCompactionTest, LevelPeriodicCompactionOffpeak) { + // This test simply checks if offpeak adjustment works in Leveled + // Compactions. For testing offpeak periodic compactions in various + // scenarios, please refer to + // DBTestUniversalCompaction2::PeriodicCompactionOffpeak + constexpr int kNumKeysPerFile = 32; + constexpr int kNumLevelFiles = 2; + constexpr int kValueSize = 100; + constexpr int kSecondsPerDay = 86400; + constexpr int kSecondsPerHour = 3600; + constexpr int kSecondsPerMinute = 60; + + for (bool if_restart : {false, true}) { + SCOPED_TRACE("if_restart=" + std::to_string(if_restart)); + Options options = CurrentOptions(); + options.ttl = 0; + options.periodic_compaction_seconds = 5 * kSecondsPerDay; // 5 days + // In the case where all files are opened and doing DB restart + // forcing the file creation time in manifest file to be 0 to + // simulate the case of reading from an old version. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionEdit::EncodeTo:VarintFileCreationTime", [&](void* arg) { + if (if_restart) { + std::string* encoded_field = static_cast(arg); + *encoded_field = ""; + PutVarint64(encoded_field, 0); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Just to add some extra random days to current time + Random rnd(test::RandomSeed()); + int days = rnd.Uniform(100); + + int periodic_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = static_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kPeriodicCompaction) { + periodic_compactions++; + } + }); + + // Starting at 12:15AM + int now_hour = 0; + int now_minute = 15; + auto mock_clock = std::make_shared(env_->GetSystemClock()); + auto mock_env = std::make_unique(env_, mock_clock); + options.env = mock_env.get(); + mock_clock->SetCurrentTime(days * kSecondsPerDay + + now_hour * kSecondsPerHour + + now_minute * kSecondsPerMinute); + // Offpeak is set from 12:30AM to 4:30AM + options.daily_offpeak_time_utc = "00:30-04:30"; + Reopen(options); + + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("2", FilesPerLevel()); + ASSERT_EQ(0, periodic_compactions); + + // Move clock forward by 1 hour. Now at 1:15AM Day 0. No compaction. + mock_clock->MockSleepForSeconds(1 * kSecondsPerHour); + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Assert that the files stay in the same level + ASSERT_EQ("3", FilesPerLevel()); + ASSERT_EQ(0, periodic_compactions); + MoveFilesToLevel(1); + ASSERT_EQ("0,3", FilesPerLevel()); + + // Move clock forward by 4 days and check if it triggers periodic + // comapaction at 1:15AM Day 4. Files created on Day 0 at 12:15AM is + // expected to expire before the offpeak starts next day at 12:30AM + mock_clock->MockSleepForSeconds(4 * kSecondsPerDay); + ASSERT_OK(Put("b", "2")); + if (if_restart) { + Reopen(options); + } else { + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("1,3", FilesPerLevel()); + // The two old files go through the periodic compaction process + ASSERT_EQ(2, periodic_compactions); + + Destroy(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + TEST_F(DBCompactionTest, LevelPeriodicCompactionWithOldDB) { // This test makes sure that periodic compactions are working with a DB // where file_creation_time of some files is 0. @@ -5231,6 +5541,12 @@ TEST_F(DBCompactionTest, CompactRangeDelayedByImmMemTableCount) { } auto manual_compaction_thread = port::Thread([this]() { + // Write something to make the current Memtable non-empty, so an extra + // immutable Memtable will be created upon manual flush requested by + // CompactRange, triggering a write stall mode to be entered because of + // accumulation of write buffers due to manual flush. + Random compact_rnd(301); + ASSERT_OK(Put(Key(0), compact_rnd.RandomString(1024))); CompactRangeOptions cro; cro.allow_write_stall = false; ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); @@ -5821,23 +6137,18 @@ TEST_P(DBCompactionDirectIOTest, DirectIO) { options.use_direct_io_for_flush_and_compaction = GetParam(); options.env = MockEnv::Create(Env::Default()); Reopen(options); - bool readahead = false; SyncPoint::GetInstance()->SetCallBack( "CompactionJob::OpenCompactionOutputFile", [&](void* arg) { bool* use_direct_writes = static_cast(arg); ASSERT_EQ(*use_direct_writes, options.use_direct_io_for_flush_and_compaction); }); - if (options.use_direct_io_for_flush_and_compaction) { - SyncPoint::GetInstance()->SetCallBack( - "SanitizeOptions:direct_io", [&](void* /*arg*/) { readahead = true; }); - } SyncPoint::GetInstance()->EnableProcessing(); CreateAndReopenWithCF({"pikachu"}, options); MakeTables(3, "p", "q", 1); ASSERT_EQ("1,1,1", FilesPerLevel(1)); Compact(1, "p", "q"); - ASSERT_EQ(readahead, options.use_direct_reads); + ASSERT_EQ(false, options.use_direct_reads); ASSERT_EQ("0,0,1", FilesPerLevel(1)); Destroy(options); delete options.env; @@ -5850,7 +6161,7 @@ class CompactionPriTest : public DBTestBase, public testing::WithParamInterface { public: CompactionPriTest() - : DBTestBase("compaction_pri_test", /*env_do_fsync=*/true) { + : DBTestBase("compaction_pri_test", /*env_do_fsync=*/false) { compaction_pri_ = GetParam(); } @@ -5970,26 +6281,30 @@ TEST_F(DBCompactionTest, PersistRoundRobinCompactCursor) { TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) { const int kKeysPerBuffer = 100; + const int kNumSubcompactions = 2; + const int kFilesPerLevel = 50; Options options = CurrentOptions(); - options.num_levels = 4; + options.num_levels = 3; options.max_bytes_for_level_multiplier = 2; options.level0_file_num_compaction_trigger = 4; options.target_file_size_base = kKeysPerBuffer * 1024; options.compaction_pri = CompactionPri::kRoundRobin; - options.max_bytes_for_level_base = 8 * kKeysPerBuffer * 1024; + // Target size is chosen so that filling the level with `kFilesPerLevel` files + // will make it oversized by `kNumSubcompactions` files. + options.max_bytes_for_level_base = + (kFilesPerLevel - kNumSubcompactions) * kKeysPerBuffer * 1024; options.disable_auto_compactions = true; - // Setup 7 threads but limited subcompactions so that - // RoundRobin requires extra compactions from reserved threads + // Setup `kNumSubcompactions` threads but limited subcompactions so + // that RoundRobin requires extra compactions from reserved threads options.max_subcompactions = 1; - options.max_background_compactions = 7; + options.max_background_compactions = kNumSubcompactions; options.max_compaction_bytes = 100000000; DestroyAndReopen(options); - env_->SetBackgroundThreads(7, Env::LOW); + env_->SetBackgroundThreads(kNumSubcompactions, Env::LOW); Random rnd(301); - const std::vector files_per_level = {0, 15, 25}; for (int lvl = 2; lvl > 0; lvl--) { - for (int i = 0; i < files_per_level[lvl]; i++) { + for (int i = 0; i < kFilesPerLevel; i++) { for (int j = 0; j < kKeysPerBuffer; j++) { // Add (lvl-1) to ensure nearly equivallent number of files // in L2 are overlapped with fils selected to compact from @@ -6000,9 +6315,8 @@ TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) { ASSERT_OK(Flush()); } MoveFilesToLevel(lvl); - ASSERT_EQ(files_per_level[lvl], NumTableFilesAtLevel(lvl, 0)); + ASSERT_EQ(kFilesPerLevel, NumTableFilesAtLevel(lvl, 0)); } - // 15 files in L1; 25 files in L2 // This is a variable for making sure the following callback is called // and the assertions in it are indeed excuted. @@ -6011,10 +6325,10 @@ TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) { "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) { uint64_t num_planned_subcompactions = *(static_cast(arg)); if (grab_pressure_token_) { - // 7 files are selected for round-robin under auto + // `kNumSubcompactions` files are selected for round-robin under auto // compaction. The number of planned subcompaction is restricted by // the limited number of max_background_compactions - ASSERT_EQ(num_planned_subcompactions, 7); + ASSERT_EQ(num_planned_subcompactions, kNumSubcompactions); } else { ASSERT_EQ(num_planned_subcompactions, 1); } @@ -9786,7 +10100,7 @@ TEST_F(DBCompactionTest, NumberOfSubcompactions) { SubCompactionEventListener* listener = new SubCompactionEventListener(); options.listeners.clear(); options.listeners.emplace_back(listener); - TryReopen(options); + ASSERT_OK(TryReopen(options)); for (int file = 0; file < kLevel0CompactTrigger; ++file) { for (int key = file; key < 2 * kNumKeyPerFile; key += 2) { @@ -9801,6 +10115,226 @@ TEST_F(DBCompactionTest, NumberOfSubcompactions) { } } +TEST_F(DBCompactionTest, VerifyRecordCount) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + options.compaction_verify_record_count = true; + DestroyAndReopen(options); + Random rnd(301); + + // Create 2 overlapping L0 files + for (int i = 1; i < 20; i += 2) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + for (int i = 0; i < 20; i += 2) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + // Only iterator through 10 keys and force compaction to finish. + int num_iter = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::ProcessKeyValueCompaction()::stop", [&](void* stop_ptr) { + num_iter++; + if (num_iter == 10) { + *(bool*)stop_ptr = true; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = db_->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_TRUE(s.IsCorruption()); + const char* expect = + "Compaction number of input keys does not match number of keys " + "processed."; + ASSERT_TRUE(std::strstr(s.getState(), expect)); +} + +TEST_F(DBCompactionTest, ErrorWhenReadFileHead) { + // This is to test a bug that is fixed in + // https://github.com/facebook/rocksdb/pull/11782. + // + // Ingest error when reading from a file with offset = 0, + // See if compaction handles it correctly. + Options opts = CurrentOptions(); + opts.num_levels = 7; + opts.compression = kNoCompression; + DestroyAndReopen(opts); + + // Set up LSM + // L5: F1 [key0, key99], F2 [key100, key199] + // L6: F3 [key50, key149] + Random rnd(301); + const int kValLen = 100; + for (int error_file = 1; error_file <= 3; ++error_file) { + for (int i = 50; i < 150; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValLen))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + + std::vector values; + for (int i = 0; i < 100; ++i) { + values.emplace_back(rnd.RandomString(kValLen)); + ASSERT_OK(Put(Key(i), values.back())); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + + for (int i = 100; i < 200; ++i) { + values.emplace_back(rnd.RandomString(kValLen)); + ASSERT_OK(Put(Key(i), values.back())); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + + ASSERT_EQ(2, NumTableFilesAtLevel(5)); + ASSERT_EQ(1, NumTableFilesAtLevel(6)); + + std::atomic_int count = 0; + SyncPoint::GetInstance()->SetCallBack( + "RandomAccessFileReader::Read::BeforeReturn", + [&count, &error_file](void* pair_ptr) { + auto p = + reinterpret_cast*>(pair_ptr); + int cur = ++count; + if (cur == error_file) { + IOStatus* io_s = p->second; + *io_s = IOStatus::IOError(); + io_s->SetRetryable(true); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = db_->CompactRange(CompactRangeOptions(), nullptr, nullptr); + // Failed compaction should not lose data. + PinnableSlice slice; + for (int i = 0; i < 200; ++i) { + ASSERT_OK(Get(Key(i), &slice)); + ASSERT_EQ(slice, values[i]); + } + ASSERT_NOK(s); + ASSERT_TRUE(s.IsIOError()); + s = db_->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_OK(s); + for (int i = 0; i < 200; ++i) { + ASSERT_OK(Get(Key(i), &slice)); + ASSERT_EQ(slice, values[i]); + } + SyncPoint::GetInstance()->DisableProcessing(); + DestroyAndReopen(opts); + } +} + +TEST_F(DBCompactionTest, ReleaseCompactionDuringManifestWrite) { + // Tests the fix for issue #10257. + // Compactions are released in LogAndApply() so that picking a compaction + // from the new Version won't see these compactions as registered. + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + // Make sure we can run multiple compactions at the same time. + env_->SetBackgroundThreads(3, Env::Priority::LOW); + env_->SetBackgroundThreads(3, Env::Priority::BOTTOM); + options.max_background_compactions = 3; + options.num_levels = 4; + DestroyAndReopen(options); + Random rnd(301); + + // Construct the following LSM + // L2: [K1-K2] [K10-K11] [k100-k101] + // L3: [K1] [K10] [k100] + // We will have 3 threads to run 3 manual compactions. + // The first thread that writes to MANIFEST will not finish + // until the next two threads enters LogAndApply() and form + // a write group. + // We check that compactions are all released after the first + // thread from the write group finishes writing to MANIFEST. + + // L3 + ASSERT_OK(Put(Key(1), rnd.RandomString(20))); + ASSERT_OK(Flush()); + MoveFilesToLevel(3); + ASSERT_OK(Put(Key(10), rnd.RandomString(20))); + ASSERT_OK(Flush()); + MoveFilesToLevel(3); + ASSERT_OK(Put(Key(100), rnd.RandomString(20))); + ASSERT_OK(Flush()); + MoveFilesToLevel(3); + // L2 + ASSERT_OK(Put(Key(100), rnd.RandomString(20))); + ASSERT_OK(Put(Key(101), rnd.RandomString(20))); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + ASSERT_OK(Put(Key(1), rnd.RandomString(20))); + ASSERT_OK(Put(Key(2), rnd.RandomString(20))); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + ASSERT_OK(Put(Key(10), rnd.RandomString(20))); + ASSERT_OK(Put(Key(11), rnd.RandomString(20))); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 3); + ASSERT_EQ(NumTableFilesAtLevel(3), 3); + + SyncPoint::GetInstance()->ClearAllCallBacks(); + std::atomic_int count = 0; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:BeforeWriterWaiting", [&](void*) { + int c = count.fetch_add(1); + if (c == 2) { + TEST_SYNC_POINT("all threads to enter LogAndApply"); + } + }); + SyncPoint::GetInstance()->LoadDependency( + {{"all threads to enter LogAndApply", + "VersionSet::LogAndApply:WriteManifestStart"}}); + // Verify that compactions are released after writing to MANIFEST + std::atomic_int after_compact_count = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:AfterCompaction", [&](void* ptr) { + int c = after_compact_count.fetch_add(1); + if (c > 0) { + ColumnFamilyData* cfd = (ColumnFamilyData*)(ptr); + ASSERT_TRUE( + cfd->compaction_picker()->compactions_in_progress()->empty()); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::vector threads; + threads.emplace_back(std::thread([&]() { + std::string k1_str = Key(1); + std::string k2_str = Key(2); + Slice k1 = k1_str; + Slice k2 = k2_str; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &k1, &k2)); + })); + threads.emplace_back(std::thread([&]() { + std::string k10_str = Key(10); + std::string k11_str = Key(11); + Slice k10 = k10_str; + Slice k11 = k11_str; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &k10, &k11)); + })); + std::string k100_str = Key(100); + std::string k101_str = Key(101); + Slice k100 = k100_str; + Slice k101 = k101_str; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &k100, &k101)); + + for (auto& thread : threads) { + thread.join(); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/db_filesnapshot.cc b/db/db_filesnapshot.cc index cb95a1676..40e7ac155 100644 --- a/db/db_filesnapshot.cc +++ b/db/db_filesnapshot.cc @@ -121,7 +121,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) { // DisableFileDeletions / EnableFileDeletions not supported in read-only DB if (deletions_disabled.ok()) { - Status s2 = EnableFileDeletions(/*force*/ false); + Status s2 = EnableFileDeletions(/*force=*/false); assert(s2.ok()); s2.PermitUncheckedError(); } else { diff --git a/db/db_flush_test.cc b/db/db_flush_test.cc index 0b2e7abb1..b2c9f4e67 100644 --- a/db/db_flush_test.cc +++ b/db/db_flush_test.cc @@ -222,7 +222,7 @@ TEST_F(DBFlushTest, CloseDBWhenFlushInLowPri) { sleeping_task_low.WaitUntilDone(); ASSERT_EQ(0, num_flushes); - TryReopenWithColumnFamilies({"default", "cf1", "cf2"}, options); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "cf1", "cf2"}, options)); ASSERT_OK(Put(0, "key3", DummyString(8192))); ASSERT_OK(Flush(0)); ASSERT_EQ(1, num_flushes); @@ -1377,6 +1377,7 @@ TEST_F(DBFlushTest, MemPurgeDeleteAndDeleteRange) { ASSERT_EQ(value, NOT_FOUND); count++; } + ASSERT_OK(iter->status()); // Expected count here is 3: KEY3, KEY4, KEY5. ASSERT_EQ(count, EXPECTED_COUNT_FORLOOP); @@ -3193,6 +3194,279 @@ INSTANTIATE_TEST_CASE_P(DBFlushDirectIOTest, DBFlushDirectIOTest, INSTANTIATE_TEST_CASE_P(DBAtomicFlushTest, DBAtomicFlushTest, testing::Bool()); +TEST_F(DBFlushTest, NonAtomicFlushRollbackPendingFlushes) { + // Fix a bug in when atomic_flush=false. + // The bug can happen as follows: + // Start Flush0 for memtable M0 to SST0 + // Start Flush1 for memtable M1 to SST1 + // Flush1 returns OK, but don't install to MANIFEST and let whoever flushes + // M0 to take care of it + // Flush0 finishes with a retryable IOError + // - It rollbacks M0, (incorrectly) not M1 + // - Deletes SST1 and SST2 + // + // Auto-recovery will start Flush2 for M0, it does not pick up M1 since it + // thinks that M1 is flushed + // Flush2 writes SST3 and finishes OK, tries to install SST3 and SST2 + // Error opening SST2 since it's already deleted + // + // The fix is to let Flush0 also rollback M1. + Options opts = CurrentOptions(); + opts.atomic_flush = false; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + opts.max_write_buffer_number = 64; + opts.max_background_flushes = 4; + env_->SetBackgroundThreads(4, Env::HIGH); + DestroyAndReopen(opts); + std::atomic_int flush_count = 0; + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) { + int c = flush_count.fetch_add(1); + if (c == 0) { + Status* s = (Status*)(s_ptr); + IOStatus io_error = IOStatus::IOError("injected foobar"); + io_error.SetRetryable(true); + *s = io_error; + TEST_SYNC_POINT("Let mem1 flush start"); + TEST_SYNC_POINT("Wait for mem1 flush to finish"); + } + }); + SyncPoint::GetInstance()->LoadDependency( + {{"Let mem1 flush start", "Mem1 flush starts"}, + {"DBImpl::BGWorkFlush:done", "Wait for mem1 flush to finish"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "Wait for error recover"}}); + // Need first flush to wait for the second flush to finish + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(Key(1), "val1")); + // trigger bg flush mem0 + ASSERT_OK(Put(Key(2), "val2")); + TEST_SYNC_POINT("Mem1 flush starts"); + // trigger bg flush mem1 + ASSERT_OK(Put(Key(3), "val3")); + + TEST_SYNC_POINT("Wait for error recover"); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBFlushTest, AbortNonAtomicFlushWhenBGError) { + // Fix a bug in when atomic_flush=false. + // The bug can happen as follows: + // Start Flush0 for memtable M0 to SST0 + // Start Flush1 for memtable M1 to SST1 + // Flush1 returns OK, but doesn't install output MANIFEST and let whoever + // flushes M0 to take care of it + // Start Flush2 for memtable M2 to SST2 + // Flush0 finishes with a retryable IOError + // - It rollbacks M0 AND M1 + // - Deletes SST1 and SST2 + // Flush2 finishes, does not rollback M2, + // - releases the pending file number that keeps SST2 alive + // - deletes SST2 + // + // Then auto-recovery starts, error opening SST2 when try to install + // flush result + // + // The fix is to let Flush2 rollback M2 if it finds that + // there is a background error. + Options opts = CurrentOptions(); + opts.atomic_flush = false; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + opts.max_write_buffer_number = 64; + opts.max_background_flushes = 4; + env_->SetBackgroundThreads(4, Env::HIGH); + DestroyAndReopen(opts); + std::atomic_int flush_count = 0; + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) { + int c = flush_count.fetch_add(1); + if (c == 0) { + Status* s = (Status*)(s_ptr); + IOStatus io_error = IOStatus::IOError("injected foobar"); + io_error.SetRetryable(true); + *s = io_error; + TEST_SYNC_POINT("Let mem1 flush start"); + TEST_SYNC_POINT("Wait for mem1 flush to finish"); + + TEST_SYNC_POINT("Let mem2 flush start"); + TEST_SYNC_POINT("Wait for mem2 to start writing table"); + } + }); + + SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table", [&](void* mems) { + autovector* mems_ptr = (autovector*)mems; + if ((*mems_ptr)[0]->GetID() == 3) { + TEST_SYNC_POINT("Mem2 flush starts writing table"); + TEST_SYNC_POINT("Mem2 flush waits until rollback"); + } + }); + SyncPoint::GetInstance()->LoadDependency( + {{"Let mem1 flush start", "Mem1 flush starts"}, + {"DBImpl::BGWorkFlush:done", "Wait for mem1 flush to finish"}, + {"Let mem2 flush start", "Mem2 flush starts"}, + {"Mem2 flush starts writing table", + "Wait for mem2 to start writing table"}, + {"RollbackMemtableFlush", "Mem2 flush waits until rollback"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "Wait for error recover"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val1")); + // trigger bg flush mem0 + ASSERT_OK(Put(Key(2), "val2")); + TEST_SYNC_POINT("Mem1 flush starts"); + // trigger bg flush mem1 + ASSERT_OK(Put(Key(3), "val3")); + + TEST_SYNC_POINT("Mem2 flush starts"); + ASSERT_OK(Put(Key(4), "val4")); + + TEST_SYNC_POINT("Wait for error recover"); + // Recovery flush writes 3 memtables together into 1 file. + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBFlushTest, NonAtomicNormalFlushAbortWhenBGError) { + Options opts = CurrentOptions(); + opts.atomic_flush = false; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + opts.max_write_buffer_number = 64; + opts.max_background_flushes = 1; + env_->SetBackgroundThreads(2, Env::HIGH); + DestroyAndReopen(opts); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + std::atomic_int flush_write_table_count = 0; + SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) { + int c = flush_write_table_count.fetch_add(1); + if (c == 0) { + Status* s = (Status*)(s_ptr); + IOStatus io_error = IOStatus::IOError("injected foobar"); + io_error.SetRetryable(true); + *s = io_error; + } + }); + + SyncPoint::GetInstance()->EnableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"Let error recovery start", + "RecoverFromRetryableBGIOError:BeforeStart"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "Wait for error recover"}}); + + ASSERT_OK(Put(Key(1), "val1")); + // trigger bg flush0 for mem0 + ASSERT_OK(Put(Key(2), "val2")); + // Not checking status since this wait can finish before flush starts. + dbfull()->TEST_WaitForFlushMemTable().PermitUncheckedError(); + + // trigger bg flush1 for mem1, should see bg error and abort + // before picking a memtable to flush + ASSERT_OK(Put(Key(3), "val3")); + ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + TEST_SYNC_POINT("Let error recovery start"); + TEST_SYNC_POINT("Wait for error recover"); + // Recovery flush writes 2 memtables together into 1 file. + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + // 1 for flush 0 and 1 for recovery flush + ASSERT_EQ(2, flush_write_table_count); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBFlushTest, DBStuckAfterAtomicFlushError) { + // Test for a bug with atomic flush where DB can become stuck + // after a flush error. A repro timeline: + // + // Start Flush0 for mem0 + // Start Flush1 for mem1 + // Now Flush1 will wait for Flush0 to install mem0 + // Flush0 finishes with retryable IOError, rollbacks mem0 + // Resume starts and waits for background job to finish, i.e., Flush1 + // Fill memtable again, trigger Flush2 for mem0 + // Flush2 will get error status, and not rollback mem0, see code in + // https://github.com/facebook/rocksdb/blob/b927ba5936216861c2c35ab68f50ba4a78e65747/db/db_impl/db_impl_compaction_flush.cc#L725 + // + // DB is stuck since mem0 can never be picked now + // + // The fix is to rollback mem0 in Flush2, and let Flush1 also abort upon + // background error besides waiting for older memtables to be installed. + // The recovery flush in this case should pick up all memtables + // and write them to a single L0 file. + Options opts = CurrentOptions(); + opts.atomic_flush = true; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + opts.max_write_buffer_number = 64; + opts.max_background_flushes = 4; + env_->SetBackgroundThreads(4, Env::HIGH); + DestroyAndReopen(opts); + + std::atomic_int flush_count = 0; + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) { + int c = flush_count.fetch_add(1); + if (c == 0) { + Status* s = (Status*)(s_ptr); + IOStatus io_error = IOStatus::IOError("injected foobar"); + io_error.SetRetryable(true); + *s = io_error; + TEST_SYNC_POINT("Let flush for mem1 start"); + // Wait for Flush1 to start waiting to install flush result + TEST_SYNC_POINT("Wait for flush for mem1"); + } + }); + SyncPoint::GetInstance()->LoadDependency( + {{"Let flush for mem1 start", "Flush for mem1"}, + {"DBImpl::AtomicFlushMemTablesToOutputFiles:WaitCV", + "Wait for flush for mem1"}, + {"RecoverFromRetryableBGIOError:BeforeStart", + "Wait for resume to start"}, + {"Recovery should continue here", + "RecoverFromRetryableBGIOError:BeforeStart2"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "Wait for error recover"}}); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(Key(1), "val1")); + // trigger Flush0 for mem0 + ASSERT_OK(Put(Key(2), "val2")); + + // trigger Flush1 for mem1 + TEST_SYNC_POINT("Flush for mem1"); + ASSERT_OK(Put(Key(3), "val3")); + + // Wait until resume started to schedule another flush + TEST_SYNC_POINT("Wait for resume to start"); + // This flush should not be scheduled due to bg error + ASSERT_OK(Put(Key(4), "val4")); + + // TEST_WaitForBackgroundWork() returns background error + // after all background work is done. + ASSERT_NOK(dbfull()->TEST_WaitForBackgroundWork()); + // Flush should abort and not writing any table + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + // Wait until this flush is done. + TEST_SYNC_POINT("Recovery should continue here"); + TEST_SYNC_POINT("Wait for error recover"); + // error recovery can schedule new flushes, but should not + // encounter error + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); +} } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/db_impl/compacted_db_impl.cc b/db/db_impl/compacted_db_impl.cc index 3d824baf2..3b665ea26 100644 --- a/db/db_impl/compacted_db_impl.cc +++ b/db/db_impl/compacted_db_impl.cc @@ -43,21 +43,34 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, /*timestamp*/ nullptr); } -Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, - const Slice& key, PinnableSlice* value, - std::string* timestamp) { - if (options.io_activity != Env::IOActivity::kUnknown) { +Status CompactedDBImpl::Get(const ReadOptions& _read_options, + ColumnFamilyHandle*, const Slice& key, + PinnableSlice* value, std::string* timestamp) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; } + assert(user_comparator_); - if (options.timestamp) { - const Status s = FailIfTsMismatchCf( - DefaultColumnFamily(), *(options.timestamp), /*ts_for_read=*/true); + if (read_options.timestamp) { + Status s = + FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp)); if (!s.ok()) { return s; } + if (read_options.timestamp->size() > 0) { + s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(), + *(read_options.timestamp)); + if (!s.ok()) { + return s; + } + } } else { const Status s = FailIfCfHasTs(DefaultColumnFamily()); if (!s.ok()) { @@ -74,7 +87,7 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, GetWithTimestampReadCallback read_cb(kMaxSequenceNumber); std::string* ts = user_comparator_->timestamp_size() > 0 ? timestamp : nullptr; - LookupKey lkey(key, kMaxSequenceNumber, options.timestamp); + LookupKey lkey(key, kMaxSequenceNumber, read_options.timestamp); GetContext get_context(user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound, lkey.user_key(), value, /*columns=*/nullptr, ts, nullptr, nullptr, true, @@ -88,8 +101,8 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, /*b_has_ts=*/false) < 0) { return Status::NotFound(); } - Status s = f.fd.table_reader->Get(options, lkey.internal_key(), &get_context, - nullptr); + Status s = f.fd.table_reader->Get(read_options, lkey.internal_key(), + &get_context, nullptr); if (!s.ok() && !s.IsNotFound()) { return s; } @@ -106,18 +119,37 @@ std::vector CompactedDBImpl::MultiGet( } std::vector CompactedDBImpl::MultiGet( - const ReadOptions& options, const std::vector&, + const ReadOptions& _read_options, const std::vector&, const std::vector& keys, std::vector* values, std::vector* timestamps) { assert(user_comparator_); size_t num_keys = keys.size(); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + return std::vector(num_keys, s); + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } - if (options.timestamp) { - Status s = FailIfTsMismatchCf(DefaultColumnFamily(), *(options.timestamp), - /*ts_for_read=*/true); + if (read_options.timestamp) { + Status s = + FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp)); if (!s.ok()) { return std::vector(num_keys, s); } + if (read_options.timestamp->size() > 0) { + s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(), + *(read_options.timestamp)); + if (!s.ok()) { + return std::vector(num_keys, s); + } + } } else { Status s = FailIfCfHasTs(DefaultColumnFamily()); if (!s.ok()) { @@ -136,7 +168,7 @@ std::vector CompactedDBImpl::MultiGet( GetWithTimestampReadCallback read_cb(kMaxSequenceNumber); autovector reader_list; for (const auto& key : keys) { - LookupKey lkey(key, kMaxSequenceNumber, options.timestamp); + LookupKey lkey(key, kMaxSequenceNumber, read_options.timestamp); const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())]; if (user_comparator_->CompareWithoutTimestamp( key, /*a_has_ts=*/false, @@ -159,14 +191,15 @@ std::vector CompactedDBImpl::MultiGet( if (r != nullptr) { PinnableSlice pinnable_val; std::string& value = (*values)[idx]; - LookupKey lkey(keys[idx], kMaxSequenceNumber, options.timestamp); + LookupKey lkey(keys[idx], kMaxSequenceNumber, read_options.timestamp); std::string* timestamp = timestamps ? &(*timestamps)[idx] : nullptr; GetContext get_context( user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound, lkey.user_key(), &pinnable_val, /*columns=*/nullptr, user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr, nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb); - Status s = r->Get(options, lkey.internal_key(), &get_context, nullptr); + Status s = + r->Get(read_options, lkey.internal_key(), &get_context, nullptr); assert(static_cast(idx) < statuses.size()); if (!s.ok() && !s.IsNotFound()) { statuses[idx] = s; diff --git a/db/db_impl/compacted_db_impl.h b/db/db_impl/compacted_db_impl.h index ad192b4f1..e1c605e42 100644 --- a/db/db_impl/compacted_db_impl.h +++ b/db/db_impl/compacted_db_impl.h @@ -30,9 +30,9 @@ class CompactedDBImpl : public DBImpl { ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) override; - Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* value, - std::string* timestamp) override; + Status Get(const ReadOptions& _read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, std::string* timestamp) override; using DB::MultiGet; // Note that CompactedDBImpl::MultiGet is not the optimized version of @@ -43,7 +43,7 @@ class CompactedDBImpl : public DBImpl { const std::vector& keys, std::vector* values) override; - std::vector MultiGet(const ReadOptions& options, + std::vector MultiGet(const ReadOptions& _read_options, const std::vector&, const std::vector& keys, std::vector* values, diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index 1e47e6dd2..78121f783 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -13,7 +13,6 @@ #include #endif -#include #include #include #include @@ -28,6 +27,7 @@ #include "db/arena_wrapped_db_iter.h" #include "db/builder.h" #include "db/compaction/compaction_job.h" +#include "db/convenience_impl.h" #include "db/db_info_dumper.h" #include "db/db_iter.h" #include "db/dbformat.h" @@ -74,7 +74,6 @@ #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/compaction_filter.h" -#include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/merge_operator.h" @@ -106,6 +105,7 @@ #include "util/mutexlock.h" #include "util/stop_watch.h" #include "util/string_util.h" +#include "util/udt_util.h" #include "utilities/trace/replayer_impl.h" namespace ROCKSDB_NAMESPACE { @@ -183,6 +183,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname, batch_per_txn_(batch_per_txn), next_job_id_(1), shutting_down_(false), + reject_new_background_jobs_(false), db_lock_(nullptr), manual_compaction_paused_(false), bg_cv_(&mutex_), @@ -209,7 +210,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname, disable_delete_obsolete_files_(0), pending_purge_obsolete_files_(0), delete_obsolete_files_last_run_(immutable_db_options_.clock->NowMicros()), - last_stats_dump_time_microsec_(0), has_unpersisted_data_(false), unable_to_release_oldest_log_(false), num_running_ingest_file_(0), @@ -272,13 +272,15 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname, periodic_task_functions_.emplace(PeriodicTaskType::kFlushInfoLog, [this]() { this->FlushInfoLog(); }); periodic_task_functions_.emplace( - PeriodicTaskType::kRecordSeqnoTime, - [this]() { this->RecordSeqnoToTimeMapping(); }); - - versions_.reset(new VersionSet(dbname_, &immutable_db_options_, file_options_, - table_cache_.get(), write_buffer_manager_, - &write_controller_, &block_cache_tracer_, - io_tracer_, db_id_, db_session_id_)); + PeriodicTaskType::kRecordSeqnoTime, [this]() { + this->RecordSeqnoToTimeMapping(/*populate_historical_seconds=*/0); + }); + + versions_.reset(new VersionSet( + dbname_, &immutable_db_options_, file_options_, table_cache_.get(), + write_buffer_manager_, &write_controller_, &block_cache_tracer_, + io_tracer_, db_id_, db_session_id_, options.daily_offpeak_time_utc, + &error_handler_)); column_family_memtables_.reset( new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet())); @@ -353,16 +355,13 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) { } // Make sure the IO Status stored in version set is set to OK. - bool file_deletion_disabled = !IsFileDeletionsEnabled(); if (s.ok()) { IOStatus io_s = versions_->io_status(); if (io_s.IsIOError()) { // If resuming from IOError resulted from MANIFEST write, then assert // that we must have already set the MANIFEST writer to nullptr during - // clean-up phase MANIFEST writing. We must have also disabled file - // deletions. + // clean-up phase MANIFEST writing. assert(!versions_->descriptor_log_); - assert(file_deletion_disabled); // Since we are trying to recover from MANIFEST write error, we need to // switch to a new MANIFEST anyway. The old MANIFEST can be corrupted. // Therefore, force writing a dummy version edit because we do not know @@ -386,27 +385,17 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) { } } - // We cannot guarantee consistency of the WAL. So force flush Memtables of - // all the column families if (s.ok()) { - FlushOptions flush_opts; - // We allow flush to stall write since we are trying to resume from error. - flush_opts.allow_write_stall = true; - if (immutable_db_options_.atomic_flush) { - mutex_.Unlock(); - s = AtomicFlushMemTables(flush_opts, context.flush_reason); - mutex_.Lock(); + if (context.flush_reason == FlushReason::kErrorRecoveryRetryFlush) { + s = RetryFlushesForErrorRecovery(FlushReason::kErrorRecoveryRetryFlush, + true /* wait */); } else { - for (auto cfd : versions_->GetRefedColumnFamilySet()) { - if (cfd->IsDropped()) { - continue; - } - InstrumentedMutexUnlock u(&mutex_); - s = FlushMemTable(cfd, flush_opts, context.flush_reason); - if (!s.ok()) { - break; - } - } + // We cannot guarantee consistency of the WAL. So force flush Memtables of + // all the column families + FlushOptions flush_opts; + // We allow flush to stall write since we are trying to resume from error. + flush_opts.allow_write_stall = true; + s = FlushAllColumnFamilies(flush_opts, context.flush_reason); } if (!s.ok()) { ROCKS_LOG_INFO(immutable_db_options_.info_log, @@ -415,34 +404,6 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) { } } - JobContext job_context(0); - FindObsoleteFiles(&job_context, true); - mutex_.Unlock(); - - job_context.manifest_file_number = 1; - if (job_context.HaveSomethingToDelete()) { - PurgeObsoleteFiles(job_context); - } - job_context.Clean(); - - if (s.ok()) { - assert(versions_->io_status().ok()); - // If we reach here, we should re-enable file deletions if it was disabled - // during previous error handling. - if (file_deletion_disabled) { - // Always return ok - s = EnableFileDeletions(/*force=*/true); - if (!s.ok()) { - ROCKS_LOG_INFO( - immutable_db_options_.info_log, - "DB resume requested but could not enable file deletions [%s]", - s.ToString().c_str()); - assert(false); - } - } - } - - mutex_.Lock(); if (s.ok()) { // This will notify and unblock threads waiting for error recovery to // finish. Those previouly waiting threads can now proceed, which may @@ -455,6 +416,15 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) { error_handler_.GetRecoveryError().PermitUncheckedError(); } + JobContext job_context(0); + FindObsoleteFiles(&job_context, true); + mutex_.Unlock(); + job_context.manifest_file_number = 1; + if (job_context.HaveSomethingToDelete()) { + PurgeObsoleteFiles(job_context); + } + job_context.Clean(); + if (s.ok()) { ROCKS_LOG_INFO(immutable_db_options_.info_log, "Successfully resumed DB"); } else { @@ -462,11 +432,31 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) { s.ToString().c_str()); } + mutex_.Lock(); // Check for shutdown again before scheduling further compactions, // since we released and re-acquired the lock above if (shutdown_initiated_) { s = Status::ShutdownInProgress(); } + if (s.ok() && context.flush_after_recovery) { + // Since we drop all non-recovery flush requests during recovery, + // and new memtable may fill up during recovery, + // schedule one more round of flush. + Status status = RetryFlushesForErrorRecovery( + FlushReason::kCatchUpAfterErrorRecovery, false /* wait */); + if (!status.ok()) { + // FlushAllColumnFamilies internally should take care of setting + // background error if needed. + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "The catch up flush after successful recovery failed [%s]", + s.ToString().c_str()); + } + // FlushAllColumnFamilies releases and re-acquires mutex. + if (shutdown_initiated_) { + s = Status::ShutdownInProgress(); + } + } + if (s.ok()) { for (auto cfd : *versions_->GetColumnFamilySet()) { SchedulePendingCompaction(cfd); @@ -494,36 +484,15 @@ void DBImpl::WaitForBackgroundWork() { void DBImpl::CancelAllBackgroundWork(bool wait) { ROCKS_LOG_INFO(immutable_db_options_.info_log, "Shutdown: canceling all background work"); - - for (uint8_t task_type = 0; - task_type < static_cast(PeriodicTaskType::kMax); task_type++) { - Status s = periodic_task_scheduler_.Unregister( - static_cast(task_type)); - if (!s.ok()) { - ROCKS_LOG_WARN(immutable_db_options_.info_log, - "Failed to unregister periodic task %d, status: %s", - task_type, s.ToString().c_str()); - } - } + Status s = CancelPeriodicTaskScheduler(); + s.PermitUncheckedError(); InstrumentedMutexLock l(&mutex_); if (!shutting_down_.load(std::memory_order_acquire) && has_unpersisted_data_.load(std::memory_order_relaxed) && !mutable_db_options_.avoid_flush_during_shutdown) { - if (immutable_db_options_.atomic_flush) { - mutex_.Unlock(); - Status s = AtomicFlushMemTables(FlushOptions(), FlushReason::kShutDown); - s.PermitUncheckedError(); //**TODO: What to do on error? - mutex_.Lock(); - } else { - for (auto cfd : versions_->GetRefedColumnFamilySet()) { - if (!cfd->IsDropped() && cfd->initialized() && !cfd->mem()->IsEmpty()) { - InstrumentedMutexUnlock u(&mutex_); - Status s = FlushMemTable(cfd, FlushOptions(), FlushReason::kShutDown); - s.PermitUncheckedError(); //**TODO: What to do on error? - } - } - } + s = DBImpl::FlushAllColumnFamilies(FlushOptions(), FlushReason::kShutDown); + s.PermitUncheckedError(); //**TODO: What to do on error? } shutting_down_.store(true, std::memory_order_release); @@ -783,7 +752,6 @@ void DBImpl::PrintStatistics() { } Status DBImpl::StartPeriodicTaskScheduler() { - #ifndef NDEBUG // It only used by test to disable scheduler bool disable_scheduler = false; @@ -827,42 +795,107 @@ Status DBImpl::StartPeriodicTaskScheduler() { return s; } -Status DBImpl::RegisterRecordSeqnoTimeWorker() { - uint64_t min_time_duration = std::numeric_limits::max(); - uint64_t max_time_duration = std::numeric_limits::min(); +Status DBImpl::RegisterRecordSeqnoTimeWorker(bool is_new_db) { + options_mutex_.AssertHeld(); + + uint64_t min_preserve_seconds = std::numeric_limits::max(); + uint64_t max_preserve_seconds = std::numeric_limits::min(); + bool mapping_was_empty = false; { InstrumentedMutexLock l(&mutex_); for (auto cfd : *versions_->GetColumnFamilySet()) { // preserve time is the max of 2 options. - uint64_t preserve_time_duration = + uint64_t preserve_seconds = std::max(cfd->ioptions()->preserve_internal_time_seconds, cfd->ioptions()->preclude_last_level_data_seconds); - if (!cfd->IsDropped() && preserve_time_duration > 0) { - min_time_duration = std::min(preserve_time_duration, min_time_duration); - max_time_duration = std::max(preserve_time_duration, max_time_duration); + if (!cfd->IsDropped() && preserve_seconds > 0) { + min_preserve_seconds = std::min(preserve_seconds, min_preserve_seconds); + max_preserve_seconds = std::max(preserve_seconds, max_preserve_seconds); } } - if (min_time_duration == std::numeric_limits::max()) { - seqno_time_mapping_.Resize(0, 0); + if (min_preserve_seconds == std::numeric_limits::max()) { + seqno_to_time_mapping_.Resize(0, 0); } else { - seqno_time_mapping_.Resize(min_time_duration, max_time_duration); + seqno_to_time_mapping_.Resize(min_preserve_seconds, max_preserve_seconds); } + mapping_was_empty = seqno_to_time_mapping_.Empty(); } uint64_t seqno_time_cadence = 0; - if (min_time_duration != std::numeric_limits::max()) { + if (min_preserve_seconds != std::numeric_limits::max()) { // round up to 1 when the time_duration is smaller than // kMaxSeqnoTimePairsPerCF - seqno_time_cadence = - (min_time_duration + SeqnoToTimeMapping::kMaxSeqnoTimePairsPerCF - 1) / - SeqnoToTimeMapping::kMaxSeqnoTimePairsPerCF; + seqno_time_cadence = (min_preserve_seconds + + SeqnoToTimeMapping::kMaxSeqnoTimePairsPerCF - 1) / + SeqnoToTimeMapping::kMaxSeqnoTimePairsPerCF; } + TEST_SYNC_POINT_CALLBACK( + "DBImpl::RegisterRecordSeqnoTimeWorker:BeforePeriodicTaskType", nullptr); + Status s; if (seqno_time_cadence == 0) { s = periodic_task_scheduler_.Unregister(PeriodicTaskType::kRecordSeqnoTime); } else { + // Before registering the periodic task, we need to be sure to fulfill two + // promises: + // 1) Any DB created with preserve/preclude options set from the beginning + // will get pre-allocated seqnos with pre-populated time mappings back to + // the times we are interested in. (This will enable future import of data + // while preserving rough write time. We can only do this reliably from + // DB::Open, as otherwise there could be a race between CreateColumnFamily + // and the first Write to the DB, and seqno-to-time mappings need to be + // monotonic. + // 2) In any DB, any data written after setting preserve/preclude options + // must have a reasonable time estimate (so that we can accurately place + // the data), which means at least one entry in seqno_to_time_mapping_. + // + // FIXME: We don't currently guarantee that if the first column family with + // that setting is added or configured after initial DB::Open but before + // the first user Write. Fixing this causes complications with the crash + // test because if DB starts without preserve/preclude option, does some + // user writes but all those writes are lost in crash, then re-opens with + // preserve/preclude option, it sees seqno==1 which looks like one of the + // user writes was recovered, when actually it was not. + bool last_seqno_zero = GetLatestSequenceNumber() == 0; + assert(!is_new_db || last_seqno_zero); + if (is_new_db && last_seqno_zero) { + // Pre-allocate seqnos and pre-populate historical mapping + assert(mapping_was_empty); + + // We can simply modify these, before writes are allowed + constexpr uint64_t kMax = SeqnoToTimeMapping::kMaxSeqnoTimePairsPerSST; + versions_->SetLastAllocatedSequence(kMax); + versions_->SetLastPublishedSequence(kMax); + versions_->SetLastSequence(kMax); + + // And record in manifest, to avoid going backwards in seqno on re-open + // (potentially with different options). Concurrency is simple because we + // are in DB::Open + { + InstrumentedMutexLock l(&mutex_); + VersionEdit edit; + edit.SetLastSequence(kMax); + s = versions_->LogAndApplyToDefaultColumnFamily( + {}, &edit, &mutex_, directories_.GetDbDir()); + if (!s.ok() && versions_->io_status().IsIOError()) { + s = error_handler_.SetBGError(versions_->io_status(), + BackgroundErrorReason::kManifestWrite); + } + } + + // Pre-populate mappings for reserved sequence numbers. + RecordSeqnoToTimeMapping(max_preserve_seconds); + } else if (mapping_was_empty) { + if (!last_seqno_zero) { + // Ensure at least one mapping (or log a warning) + RecordSeqnoToTimeMapping(/*populate_historical_seconds=*/0); + } else { + // FIXME (see limitation described above) + } + } + s = periodic_task_scheduler_.Register( PeriodicTaskType::kRecordSeqnoTime, periodic_task_functions_.at(PeriodicTaskType::kRecordSeqnoTime), @@ -872,8 +905,24 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker() { return s; } +Status DBImpl::CancelPeriodicTaskScheduler() { + Status s = Status::OK(); + for (uint8_t task_type = 0; + task_type < static_cast(PeriodicTaskType::kMax); task_type++) { + s = periodic_task_scheduler_.Unregister( + static_cast(task_type)); + if (!s.ok()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Failed to unregister periodic task %d, status: %s", + task_type, s.ToString().c_str()); + } + } + return s; +} + // esitmate the total size of stats_history_ size_t DBImpl::EstimateInMemoryStatsHistorySize() const { + stats_history_mutex_.AssertHeld(); size_t size_total = sizeof(std::map>); if (stats_history_.size() == 0) return size_total; @@ -970,7 +1019,7 @@ void DBImpl::PersistStats() { "Storing %" ROCKSDB_PRIszt " stats with timestamp %" PRIu64 " to in-memory stats history", stats_slice_.size(), now_seconds); - stats_history_[now_seconds] = stats_delta; + stats_history_[now_seconds] = std::move(stats_delta); } stats_slice_initialized_ = true; std::swap(stats_slice_, stats_map); @@ -1164,6 +1213,7 @@ Status DBImpl::SetOptions( return Status::InvalidArgument("empty input"); } + InstrumentedMutexLock ol(&options_mutex_); MutableCFOptions new_options; Status s; Status persist_options_status; @@ -1183,8 +1233,7 @@ Status DBImpl::SetOptions( // thread. InstallSuperVersionAndScheduleWork(cfd, &sv_context, new_options); - persist_options_status = WriteOptionsFile( - false /*need_mutex_lock*/, true /*need_enter_write_thread*/); + persist_options_status = WriteOptionsFile(true /*db_mutex_already_held*/); bg_cv_.SignalAll(); } } @@ -1222,10 +1271,12 @@ Status DBImpl::SetDBOptions( return Status::InvalidArgument("empty input"); } + InstrumentedMutexLock ol(&options_mutex_); MutableDBOptions new_options; Status s; Status persist_options_status = Status::OK(); - bool wal_changed = false; + bool wal_size_option_changed = false; + bool wal_other_option_changed = false; WriteContext write_context; { InstrumentedMutexLock l(&mutex_); @@ -1276,17 +1327,24 @@ Status DBImpl::SetDBOptions( const bool max_compactions_increased = new_bg_job_limits.max_compactions > current_bg_job_limits.max_compactions; + const bool offpeak_time_changed = + versions_->offpeak_time_option().daily_offpeak_time_utc != + new_db_options.daily_offpeak_time_utc; - if (max_flushes_increased || max_compactions_increased) { + if (max_flushes_increased || max_compactions_increased || + offpeak_time_changed) { if (max_flushes_increased) { env_->IncBackgroundThreadsIfNeeded(new_bg_job_limits.max_flushes, Env::Priority::HIGH); } - if (max_compactions_increased) { env_->IncBackgroundThreadsIfNeeded(new_bg_job_limits.max_compactions, Env::Priority::LOW); } + if (offpeak_time_changed) { + versions_->ChangeOffpeakTimeOption( + new_db_options.daily_offpeak_time_utc); + } MaybeScheduleFlushOrCompaction(); } @@ -1326,8 +1384,10 @@ Status DBImpl::SetDBOptions( table_cache_.get()->SetCapacity(new_options.max_open_files == -1 ? TableCache::kInfiniteCapacity : new_options.max_open_files - 10); - wal_changed = mutable_db_options_.wal_bytes_per_sync != - new_options.wal_bytes_per_sync; + wal_other_option_changed = mutable_db_options_.wal_bytes_per_sync != + new_options.wal_bytes_per_sync; + wal_size_option_changed = mutable_db_options_.max_total_wal_size != + new_options.max_total_wal_size; mutable_db_options_ = new_options; file_options_for_compaction_ = FileOptions(new_db_options); file_options_for_compaction_ = fs_->OptimizeForCompactionTableWrite( @@ -1338,19 +1398,21 @@ Status DBImpl::SetDBOptions( file_options_for_compaction_, immutable_db_options_); file_options_for_compaction_.compaction_readahead_size = mutable_db_options_.compaction_readahead_size; - WriteThread::Writer w; - write_thread_.EnterUnbatched(&w, &mutex_); - if (total_log_size_ > GetMaxTotalWalSize() || wal_changed) { - Status purge_wal_status = SwitchWAL(&write_context); - if (!purge_wal_status.ok()) { - ROCKS_LOG_WARN(immutable_db_options_.info_log, - "Unable to purge WAL files in SetDBOptions() -- %s", - purge_wal_status.ToString().c_str()); + if (wal_other_option_changed || wal_size_option_changed) { + WriteThread::Writer w; + write_thread_.EnterUnbatched(&w, &mutex_); + if (wal_other_option_changed || + total_log_size_ > GetMaxTotalWalSize()) { + Status purge_wal_status = SwitchWAL(&write_context); + if (!purge_wal_status.ok()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Unable to purge WAL files in SetDBOptions() -- %s", + purge_wal_status.ToString().c_str()); + } } + write_thread_.ExitUnbatched(&w); } - persist_options_status = WriteOptionsFile( - false /*need_mutex_lock*/, false /*need_enter_write_thread*/); - write_thread_.ExitUnbatched(&w); + persist_options_status = WriteOptionsFile(true /*db_mutex_already_held*/); } else { // To get here, we must have had invalid options and will not attempt to // persist the options, which means the status is "OK/Uninitialized. @@ -1695,7 +1757,8 @@ Status DBImpl::GetFullHistoryTsLow(ColumnFamilyHandle* column_family, } InstrumentedMutexLock l(&mutex_); *ts_low = cfd->GetFullHistoryTsLow(); - assert(cfd->user_comparator()->timestamp_size() == ts_low->size()); + assert(ts_low->empty() || + cfd->user_comparator()->timestamp_size() == ts_low->size()); return Status::OK(); } @@ -1922,38 +1985,68 @@ Status DBImpl::Get(const ReadOptions& read_options, return Get(read_options, column_family, key, value, /*timestamp=*/nullptr); } -Status DBImpl::Get(const ReadOptions& read_options, +Status DBImpl::GetImpl(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) { + return GetImpl(read_options, column_family, key, value, + /*timestamp=*/nullptr); +} + +Status DBImpl::Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value, std::string* timestamp) { assert(value != nullptr); value->Reset(); + + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { + return Status::InvalidArgument( + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; + } + + Status s = GetImpl(read_options, column_family, key, value, timestamp); + return s; +} + +Status DBImpl::GetImpl(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, std::string* timestamp) { GetImplOptions get_impl_options; get_impl_options.column_family = column_family; get_impl_options.value = value; get_impl_options.timestamp = timestamp; + Status s = GetImpl(read_options, key, get_impl_options); return s; } -Status DBImpl::GetEntity(const ReadOptions& read_options, +Status DBImpl::GetEntity(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableWideColumns* columns) { if (!column_family) { return Status::InvalidArgument( "Cannot call GetEntity without a column family handle"); } - if (!columns) { return Status::InvalidArgument( "Cannot call GetEntity without a PinnableWideColumns object"); } - - if (read_options.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGetEntity) { return Status::InvalidArgument( "Cannot call GetEntity with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGetEntity`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGetEntity; } - columns->Reset(); GetImplOptions get_impl_options; @@ -1963,6 +2056,73 @@ Status DBImpl::GetEntity(const ReadOptions& read_options, return GetImpl(read_options, key, get_impl_options); } +Status DBImpl::GetEntity(const ReadOptions& _read_options, const Slice& key, + PinnableAttributeGroups* result) { + if (!result) { + return Status::InvalidArgument( + "Cannot call GetEntity without PinnableAttributeGroups object"); + } + Status s; + const size_t num_column_families = result->size(); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGetEntity) { + s = Status::InvalidArgument( + "Cannot call GetEntity with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGetEntity`"); + for (size_t i = 0; i < num_column_families; ++i) { + (*result)[i].SetStatus(s); + } + return s; + } + // return early if no CF was passed in + if (num_column_families == 0) { + return s; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGetEntity; + } + std::vector keys; + std::vector column_families; + for (size_t i = 0; i < num_column_families; ++i) { + // If any of the CFH is null, break early since the entire query will fail + if (!(*result)[i].column_family()) { + s = Status::InvalidArgument( + "DB failed to query because one or more group(s) have null column " + "family handle"); + (*result)[i].SetStatus( + Status::InvalidArgument("Column family handle cannot be null")); + break; + } + // Adding the same key slice for different CFs + keys.emplace_back(key); + column_families.emplace_back((*result)[i].column_family()); + } + if (!s.ok()) { + for (size_t i = 0; i < num_column_families; ++i) { + if ((*result)[i].status().ok()) { + (*result)[i].SetStatus( + Status::Incomplete("DB not queried due to invalid argument(s) in " + "one or more of the attribute groups")); + } + } + return s; + } + std::vector columns(num_column_families); + std::vector statuses(num_column_families); + MultiGetCommon( + read_options, num_column_families, column_families.data(), keys.data(), + /* values */ nullptr, columns.data(), + /* timestamps */ nullptr, statuses.data(), /* sorted_input */ false); + // Set results + for (size_t i = 0; i < num_column_families; ++i) { + (*result)[i].Reset(); + (*result)[i].SetStatus(statuses[i]); + (*result)[i].SetColumns(std::move(columns[i])); + } + return s; +} + bool DBImpl::ShouldReferenceSuperVersion(const MergeContext& merge_context) { // If both thresholds are reached, a function returning merge operands as // `PinnableSlice`s should reference the `SuperVersion` to avoid large and/or @@ -1999,16 +2159,9 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, assert(get_impl_options.column_family); - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - } - if (read_options.timestamp) { const Status s = FailIfTsMismatchCf(get_impl_options.column_family, - *(read_options.timestamp), - /*ts_for_read=*/true); + *(read_options.timestamp)); if (!s.ok()) { return s; } @@ -2055,7 +2208,16 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, // Acquire SuperVersion SuperVersion* sv = GetAndRefSuperVersion(cfd); + if (read_options.timestamp && read_options.timestamp->size() > 0) { + const Status s = + FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp)); + if (!s.ok()) { + ReturnAndCleanupSuperVersion(cfd, sv); + return s; + } + } + TEST_SYNC_POINT_CALLBACK("DBImpl::GetImpl:AfterAcquireSv", nullptr); TEST_SYNC_POINT("DBImpl::GetImpl:1"); TEST_SYNC_POINT("DBImpl::GetImpl:2"); @@ -2206,6 +2368,12 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, RecordTick(stats_, NUMBER_KEYS_READ); size_t size = 0; if (s.ok()) { + const auto& merge_threshold = read_options.merge_operand_count_threshold; + if (merge_threshold.has_value() && + merge_context.GetNumOperands() > merge_threshold.value()) { + s = Status::OkMergeOperandThresholdExceeded(); + } + if (get_impl_options.get_value) { if (get_impl_options.value) { size = get_impl_options.value->size(); @@ -2297,7 +2465,7 @@ std::vector DBImpl::MultiGet( } std::vector DBImpl::MultiGet( - const ReadOptions& read_options, + const ReadOptions& _read_options, const std::vector& column_family, const std::vector& keys, std::vector* values, std::vector* timestamps) { @@ -2309,12 +2477,29 @@ std::vector DBImpl::MultiGet( assert(column_family.size() == num_keys); std::vector stat_list(num_keys); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + + for (size_t i = 0; i < num_keys; ++i) { + stat_list[i] = s; + } + return stat_list; + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } + bool should_fail = false; for (size_t i = 0; i < num_keys; ++i) { assert(column_family[i]); if (read_options.timestamp) { - stat_list[i] = FailIfTsMismatchCf( - column_family[i], *(read_options.timestamp), /*ts_for_read=*/true); + stat_list[i] = + FailIfTsMismatchCf(column_family[i], *(read_options.timestamp)); if (!stat_list[i].ok()) { should_fail = true; } @@ -2346,8 +2531,6 @@ std::vector DBImpl::MultiGet( } } - SequenceNumber consistent_seqnum; - UnorderedMap multiget_cf_data( column_family.size()); for (auto cf : column_family) { @@ -2365,10 +2548,21 @@ std::vector DBImpl::MultiGet( [](UnorderedMap::iterator& cf_iter) { return &cf_iter->second; }; - bool unref_only = + SequenceNumber consistent_seqnum; + bool sv_from_thread_local; + Status status = MultiCFSnapshot>( read_options, nullptr, iter_deref_lambda, &multiget_cf_data, - &consistent_seqnum); + &consistent_seqnum, &sv_from_thread_local); + + if (!status.ok()) { + for (auto& s : stat_list) { + if (s.ok()) { + s = status; + } + } + return stat_list; + } TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum1"); TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum2"); @@ -2449,8 +2643,15 @@ std::vector DBImpl::MultiGet( } if (s.ok()) { + const auto& merge_threshold = read_options.merge_operand_count_threshold; + if (merge_threshold.has_value() && + merge_context.GetNumOperands() > merge_threshold.value()) { + s = Status::OkMergeOperandThresholdExceeded(); + } + bytes_read += value->size(); num_found++; + curr_value_size += value->size(); if (curr_value_size > read_options.value_size_soft_limit) { while (++keys_read < num_keys) { @@ -2481,10 +2682,11 @@ std::vector DBImpl::MultiGet( for (auto mgd_iter : multiget_cf_data) { auto mgd = mgd_iter.second; - if (!unref_only) { + if (sv_from_thread_local) { ReturnAndCleanupSuperVersion(mgd.cfd, mgd.super_version); } else { - mgd.cfd->GetSuperVersion()->Unref(); + TEST_SYNC_POINT("DBImpl::MultiGet::BeforeLastTryUnRefSV"); + CleanupSuperVersion(mgd.super_version); } } RecordTick(stats_, NUMBER_MULTIGET_CALLS); @@ -2499,21 +2701,49 @@ std::vector DBImpl::MultiGet( } template -bool DBImpl::MultiCFSnapshot( +Status DBImpl::MultiCFSnapshot( const ReadOptions& read_options, ReadCallback* callback, std::function& iter_deref_func, - T* cf_list, SequenceNumber* snapshot) { + T* cf_list, SequenceNumber* snapshot, bool* sv_from_thread_local) { PERF_TIMER_GUARD(get_snapshot_time); + assert(sv_from_thread_local); + *sv_from_thread_local = true; + Status s = Status::OK(); + const bool check_read_ts = + read_options.timestamp && read_options.timestamp->size() > 0; + // sv_from_thread_local set to false means the SuperVersion to be cleaned up + // is acquired directly via ColumnFamilyData instead of thread local. + const auto sv_cleanup_func = [&]() -> void { + for (auto cf_iter = cf_list->begin(); cf_iter != cf_list->end(); + ++cf_iter) { + auto node = iter_deref_func(cf_iter); + SuperVersion* super_version = node->super_version; + ColumnFamilyData* cfd = node->cfd; + if (super_version != nullptr) { + if (*sv_from_thread_local) { + ReturnAndCleanupSuperVersion(cfd, super_version); + } else { + CleanupSuperVersion(super_version); + } + } + node->super_version = nullptr; + } + }; + bool last_try = false; if (cf_list->size() == 1) { - // Fast path for a single column family. We can simply get the thread loca + // Fast path for a single column family. We can simply get the thread local // super version auto cf_iter = cf_list->begin(); auto node = iter_deref_func(cf_iter); node->super_version = GetAndRefSuperVersion(node->cfd); - if (read_options.snapshot != nullptr) { + if (check_read_ts) { + s = FailIfReadCollapsedHistory(node->cfd, node->super_version, + *(read_options.timestamp)); + } + if (s.ok() && read_options.snapshot != nullptr) { // Note: In WritePrepared txns this is not necessary but not harmful // either. Because prep_seq > snapshot => commit_seq > snapshot so if // a snapshot is specified we should be fine with skipping seq numbers @@ -2527,7 +2757,7 @@ bool DBImpl::MultiCFSnapshot( if (callback) { *snapshot = std::max(*snapshot, callback->max_visible_seq()); } - } else { + } else if (s.ok()) { // Since we get and reference the super version before getting // the snapshot number, without a mutex protection, it is possible // that a memtable switch happened in the middle and not all the @@ -2541,26 +2771,17 @@ bool DBImpl::MultiCFSnapshot( *snapshot = GetLastPublishedSequence(); } } else { - // If we end up with the same issue of memtable geting sealed during 2 + // If we end up with the same issue of memtable getting sealed during 2 // consecutive retries, it means the write rate is very high. In that case - // its probably ok to take the mutex on the 3rd try so we can succeed for - // sure + // it's probably ok to take the mutex on the 3rd try so we can succeed for + // sure. constexpr int num_retries = 3; for (int i = 0; i < num_retries; ++i) { last_try = (i == num_retries - 1); bool retry = false; if (i > 0) { - for (auto cf_iter = cf_list->begin(); cf_iter != cf_list->end(); - ++cf_iter) { - auto node = iter_deref_func(cf_iter); - SuperVersion* super_version = node->super_version; - ColumnFamilyData* cfd = node->cfd; - if (super_version != nullptr) { - ReturnAndCleanupSuperVersion(cfd, super_version); - } - node->super_version = nullptr; - } + sv_cleanup_func(); } if (read_options.snapshot == nullptr) { if (last_try) { @@ -2584,6 +2805,19 @@ bool DBImpl::MultiCFSnapshot( node->super_version = node->cfd->GetSuperVersion()->Ref(); } TEST_SYNC_POINT("DBImpl::MultiGet::AfterRefSV"); + if (check_read_ts) { + s = FailIfReadCollapsedHistory(node->cfd, node->super_version, + *(read_options.timestamp)); + if (!s.ok()) { + // If read timestamp check failed, a.k.a ReadOptions.timestamp < + // super_version.full_history_ts_low. There is no need to continue + // because this check will keep failing for the same and newer + // SuperVersions, instead we fail fast and ask user to provide + // a higher read timestamp. + retry = false; + break; + } + } if (read_options.snapshot != nullptr || last_try) { // If user passed a snapshot, then we don't care if a memtable is // sealed or compaction happens because the snapshot would ensure @@ -2607,6 +2841,7 @@ bool DBImpl::MultiCFSnapshot( if (!retry) { if (last_try) { mutex_.Unlock(); + TEST_SYNC_POINT("DBImpl::MultiGet::AfterLastTryRefSV"); } break; } @@ -2615,8 +2850,11 @@ bool DBImpl::MultiCFSnapshot( // Keep track of bytes that we read for statistics-recording later PERF_TIMER_STOP(get_snapshot_time); - - return last_try; + *sv_from_thread_local = !last_try; + if (!s.ok()) { + sv_cleanup_func(); + } + return s; } void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys, @@ -2627,10 +2865,26 @@ void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys, /* timestamps */ nullptr, statuses, sorted_input); } -void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys, +void DBImpl::MultiGet(const ReadOptions& _read_options, const size_t num_keys, ColumnFamilyHandle** column_families, const Slice* keys, PinnableSlice* values, std::string* timestamps, Status* statuses, const bool sorted_input) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } MultiGetCommon(read_options, num_keys, column_families, keys, values, /* columns */ nullptr, timestamps, statuses, sorted_input); } @@ -2645,14 +2899,11 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, if (num_keys == 0) { return; } - bool should_fail = false; for (size_t i = 0; i < num_keys; ++i) { ColumnFamilyHandle* cfh = column_families[i]; - assert(cfh); if (read_options.timestamp) { - statuses[i] = FailIfTsMismatchCf(cfh, *(read_options.timestamp), - /*ts_for_read=*/true); + statuses[i] = FailIfTsMismatchCf(cfh, *(read_options.timestamp)); if (!statuses[i].ok()) { should_fail = true; } @@ -2735,10 +2986,20 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, }; SequenceNumber consistent_seqnum; - bool unref_only = MultiCFSnapshot< + bool sv_from_thread_local; + Status s = MultiCFSnapshot< autovector>( read_options, nullptr, iter_deref_lambda, &multiget_cf_data, - &consistent_seqnum); + &consistent_seqnum, &sv_from_thread_local); + + if (!s.ok()) { + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } GetWithTimestampReadCallback timestamp_read_callback(0); ReadCallback* read_callback = nullptr; @@ -2747,7 +3008,6 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, read_callback = ×tamp_read_callback; } - Status s; auto cf_iter = multiget_cf_data.begin(); for (; cf_iter != multiget_cf_data.end(); ++cf_iter) { s = MultiGetImpl(read_options, cf_iter->start, cf_iter->num_keys, @@ -2768,10 +3028,11 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, } for (const auto& iter : multiget_cf_data) { - if (!unref_only) { + if (sv_from_thread_local) { ReturnAndCleanupSuperVersion(iter.cfd, iter.super_version); } else { - iter.cfd->GetSuperVersion()->Unref(); + TEST_SYNC_POINT("DBImpl::MultiGet::BeforeLastTryUnRefSV"); + CleanupSuperVersion(iter.super_version); } } } @@ -2828,11 +3089,28 @@ void DBImpl::MultiGet(const ReadOptions& read_options, /* timestamps */ nullptr, statuses, sorted_input); } -void DBImpl::MultiGet(const ReadOptions& read_options, +void DBImpl::MultiGet(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, std::string* timestamps, Status* statuses, const bool sorted_input) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } MultiGetCommon(read_options, column_family, num_keys, keys, values, /* columns */ nullptr, timestamps, statuses, sorted_input); } @@ -2877,10 +3155,27 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, sorted_keys[i] = &key_context[i]; } PrepareMultiGetKeys(num_keys, sorted_input, &sorted_keys); - MultiGetWithCallback(read_options, column_family, nullptr, &sorted_keys); + MultiGetWithCallbackImpl(read_options, column_family, nullptr, &sorted_keys); } void DBImpl::MultiGetWithCallback( + const ReadOptions& _read_options, ColumnFamilyHandle* column_family, + ReadCallback* callback, + autovector* sorted_keys) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + assert(false); + return; + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } + MultiGetWithCallbackImpl(read_options, column_family, callback, sorted_keys); +} + +void DBImpl::MultiGetWithCallbackImpl( const ReadOptions& read_options, ColumnFamilyHandle* column_family, ReadCallback* callback, autovector* sorted_keys) { @@ -2895,14 +3190,18 @@ void DBImpl::MultiGetWithCallback( size_t num_keys = sorted_keys->size(); SequenceNumber consistent_seqnum; - bool unref_only = MultiCFSnapshot>( + bool sv_from_thread_local; + Status s = MultiCFSnapshot>( read_options, callback, iter_deref_lambda, &multiget_cf_data, - &consistent_seqnum); + &consistent_seqnum, &sv_from_thread_local); + if (!s.ok()) { + return; + } #ifndef NDEBUG - assert(!unref_only); + assert(sv_from_thread_local); #else // Silence unused variable warning - (void)unref_only; + (void)sv_from_thread_local; #endif // NDEBUG if (callback && read_options.snapshot == nullptr) { @@ -2932,9 +3231,9 @@ void DBImpl::MultiGetWithCallback( read_callback = ×tamp_read_callback; } - Status s = MultiGetImpl(read_options, 0, num_keys, sorted_keys, - multiget_cf_data[0].super_version, consistent_seqnum, - read_callback); + s = MultiGetImpl(read_options, 0, num_keys, sorted_keys, + multiget_cf_data[0].super_version, consistent_seqnum, + read_callback); assert(s.ok() || s.IsTimedOut() || s.IsAborted()); ReturnAndCleanupSuperVersion(multiget_cf_data[0].cfd, multiget_cf_data[0].super_version); @@ -2954,11 +3253,6 @@ Status DBImpl::MultiGetImpl( autovector* sorted_keys, SuperVersion* super_version, SequenceNumber snapshot, ReadCallback* callback) { - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return Status::InvalidArgument( - "Cannot call MultiGet with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - } PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); StopWatch sw(immutable_db_options_.clock, stats_, DB_MULTIGET); @@ -2995,7 +3289,7 @@ Status DBImpl::MultiGetImpl( stats_); MultiGetRange range = ctx.GetMultiGetRange(); range.AddValueSize(curr_value_size); - bool lookup_current = false; + bool lookup_current = true; keys_left -= batch_size; for (auto mget_iter = range.begin(); mget_iter != range.end(); @@ -3014,9 +3308,10 @@ Status DBImpl::MultiGetImpl( super_version->imm->MultiGet(read_options, &range, callback); } if (!range.empty()) { - lookup_current = true; uint64_t left = range.KeysLeft(); RecordTick(stats_, MEMTABLE_MISS, left); + } else { + lookup_current = false; } } if (lookup_current) { @@ -3040,6 +3335,12 @@ Status DBImpl::MultiGetImpl( assert(key->s); if (key->s->ok()) { + const auto& merge_threshold = read_options.merge_operand_count_threshold; + if (merge_threshold.has_value() && + key->merge_context.GetNumOperands() > merge_threshold) { + *(key->s) = Status::OkMergeOperandThresholdExceeded(); + } + if (key->value) { bytes_read += key->value->size(); } else { @@ -3070,30 +3371,136 @@ Status DBImpl::MultiGetImpl( return s; } -void DBImpl::MultiGetEntity(const ReadOptions& options, size_t num_keys, +void DBImpl::MultiGetEntity(const ReadOptions& _read_options, size_t num_keys, ColumnFamilyHandle** column_families, const Slice* keys, PinnableWideColumns* results, Status* statuses, bool sorted_input) { - MultiGetCommon(options, num_keys, column_families, keys, /* values */ nullptr, - results, /* timestamps */ nullptr, statuses, sorted_input); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGetEntity) { + Status s = Status::InvalidArgument( + "Can only call MultiGetEntity with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGetEntity`"); + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGetEntity; + } + MultiGetCommon(read_options, num_keys, column_families, keys, + /* values */ nullptr, results, /* timestamps */ nullptr, + statuses, sorted_input); } -void DBImpl::MultiGetEntity(const ReadOptions& options, +void DBImpl::MultiGetEntity(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, size_t num_keys, const Slice* keys, PinnableWideColumns* results, Status* statuses, bool sorted_input) { - MultiGetCommon(options, column_family, num_keys, keys, /* values */ nullptr, - results, /* timestamps */ nullptr, statuses, sorted_input); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGetEntity) { + Status s = Status::InvalidArgument( + "Can only call MultiGetEntity with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGetEntity`"); + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGetEntity; + } + MultiGetCommon(read_options, column_family, num_keys, keys, + /* values */ nullptr, results, /* timestamps */ nullptr, + statuses, sorted_input); +} + +void DBImpl::MultiGetEntity(const ReadOptions& _read_options, size_t num_keys, + const Slice* keys, + PinnableAttributeGroups* results) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGetEntity) { + Status s = Status::InvalidArgument( + "Can only call MultiGetEntity with ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGetEntity`"); + for (size_t i = 0; i < num_keys; ++i) { + for (size_t j = 0; j < results[i].size(); ++j) { + results[i][j].SetStatus(s); + } + } + return; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGetEntity; + } + + std::vector column_families; + std::vector all_keys; + size_t total_count = 0; + + for (size_t i = 0; i < num_keys; ++i) { + for (size_t j = 0; j < results[i].size(); ++j) { + // Adding the same key slice for different CFs + all_keys.emplace_back(keys[i]); + column_families.emplace_back(results[i][j].column_family()); + ++total_count; + } + } + std::vector statuses(total_count); + std::vector columns(total_count); + MultiGetCommon(read_options, total_count, column_families.data(), + all_keys.data(), + /* values */ nullptr, columns.data(), + /* timestamps */ nullptr, statuses.data(), + /* sorted_input */ false); + + // Set results + size_t index = 0; + for (size_t i = 0; i < num_keys; ++i) { + for (size_t j = 0; j < results[i].size(); ++j) { + results[i][j].Reset(); + results[i][j].SetStatus(std::move(statuses[index])); + results[i][j].SetColumns(std::move(columns[index])); + ++index; + } + } +} + +Status DBImpl::WrapUpCreateColumnFamilies( + const std::vector& cf_options) { + // NOTE: this function is skipped for create_missing_column_families and + // DB::Open, so new functionality here might need to go into Open also. + bool register_worker = false; + for (auto* opts_ptr : cf_options) { + if (opts_ptr->preserve_internal_time_seconds > 0 || + opts_ptr->preclude_last_level_data_seconds > 0) { + register_worker = true; + break; + } + } + // Attempt both follow-up actions even if one fails + Status s = WriteOptionsFile(false /*db_mutex_already_held*/); + if (register_worker) { + s.UpdateIfOk(RegisterRecordSeqnoTimeWorker(/*from_db_open=*/false)); + } + return s; } Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options, const std::string& column_family, ColumnFamilyHandle** handle) { assert(handle != nullptr); + InstrumentedMutexLock ol(&options_mutex_); Status s = CreateColumnFamilyImpl(cf_options, column_family, handle); if (s.ok()) { - s = WriteOptionsFile(true /*need_mutex_lock*/, - true /*need_enter_write_thread*/); + s.UpdateIfOk(WrapUpCreateColumnFamilies({&cf_options})); } return s; } @@ -3103,6 +3510,7 @@ Status DBImpl::CreateColumnFamilies( const std::vector& column_family_names, std::vector* handles) { assert(handles != nullptr); + InstrumentedMutexLock ol(&options_mutex_); handles->clear(); size_t num_cf = column_family_names.size(); Status s; @@ -3117,11 +3525,7 @@ Status DBImpl::CreateColumnFamilies( success_once = true; } if (success_once) { - Status persist_options_status = WriteOptionsFile( - true /*need_mutex_lock*/, true /*need_enter_write_thread*/); - if (s.ok() && !persist_options_status.ok()) { - s = persist_options_status; - } + s.UpdateIfOk(WrapUpCreateColumnFamilies({&cf_options})); } return s; } @@ -3130,10 +3534,13 @@ Status DBImpl::CreateColumnFamilies( const std::vector& column_families, std::vector* handles) { assert(handles != nullptr); + InstrumentedMutexLock ol(&options_mutex_); handles->clear(); size_t num_cf = column_families.size(); Status s; bool success_once = false; + std::vector cf_opts; + cf_opts.reserve(num_cf); for (size_t i = 0; i < num_cf; i++) { ColumnFamilyHandle* handle; s = CreateColumnFamilyImpl(column_families[i].options, @@ -3143,13 +3550,10 @@ Status DBImpl::CreateColumnFamilies( } handles->push_back(handle); success_once = true; + cf_opts.push_back(&column_families[i].options); } if (success_once) { - Status persist_options_status = WriteOptionsFile( - true /*need_mutex_lock*/, true /*need_enter_write_thread*/); - if (s.ok() && !persist_options_status.ok()) { - s = persist_options_status; - } + s.UpdateIfOk(WrapUpCreateColumnFamilies(cf_opts)); } return s; } @@ -3157,6 +3561,7 @@ Status DBImpl::CreateColumnFamilies( Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options, const std::string& column_family_name, ColumnFamilyHandle** handle) { + options_mutex_.AssertHeld(); // TODO: plumb Env::IOActivity const ReadOptions read_options; Status s; @@ -3191,6 +3596,8 @@ Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options, edit.SetColumnFamily(new_id); edit.SetLogNumber(logfile_number_); edit.SetComparatorName(cf_options.comparator->Name()); + edit.SetPersistUserDefinedTimestamps( + cf_options.persist_user_defined_timestamps); // LogAndApply will both write the creation in MANIFEST and create // ColumnFamilyData object @@ -3235,10 +3642,6 @@ Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options, } } // InstrumentedMutexLock l(&mutex_) - if (cf_options.preserve_internal_time_seconds > 0 || - cf_options.preclude_last_level_data_seconds > 0) { - s = RegisterRecordSeqnoTimeWorker(); - } sv_context.Clean(); // this is outside the mutex if (s.ok()) { @@ -3250,16 +3653,17 @@ Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options, Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) { assert(column_family != nullptr); + InstrumentedMutexLock ol(&options_mutex_); Status s = DropColumnFamilyImpl(column_family); if (s.ok()) { - s = WriteOptionsFile(true /*need_mutex_lock*/, - true /*need_enter_write_thread*/); + s = WriteOptionsFile(false /*db_mutex_already_held*/); } return s; } Status DBImpl::DropColumnFamilies( const std::vector& column_families) { + InstrumentedMutexLock ol(&options_mutex_); Status s; bool success_once = false; for (auto* handle : column_families) { @@ -3270,8 +3674,8 @@ Status DBImpl::DropColumnFamilies( success_once = true; } if (success_once) { - Status persist_options_status = WriteOptionsFile( - true /*need_mutex_lock*/, true /*need_enter_write_thread*/); + Status persist_options_status = + WriteOptionsFile(false /*db_mutex_already_held*/); if (s.ok() && !persist_options_status.ok()) { s = persist_options_status; } @@ -3332,7 +3736,7 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) { if (cfd->ioptions()->preserve_internal_time_seconds > 0 || cfd->ioptions()->preclude_last_level_data_seconds > 0) { - s = RegisterRecordSeqnoTimeWorker(); + s = RegisterRecordSeqnoTimeWorker(/*from_db_open=*/false); } if (s.ok()) { @@ -3381,8 +3785,19 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options, return s.ok() || s.IsIncomplete(); } -Iterator* DBImpl::NewIterator(const ReadOptions& read_options, +Iterator* DBImpl::NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { + return NewErrorIterator(Status::InvalidArgument( + "Can only call NewIterator with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`")); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; + } + if (read_options.managed) { return NewErrorIterator( Status::NotSupported("Managed iterator is not supported anymore.")); @@ -3392,17 +3807,11 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options, return NewErrorIterator(Status::NotSupported( "ReadTier::kPersistedData is not yet supported in iterators.")); } - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return NewErrorIterator(Status::InvalidArgument( - "Cannot call NewIterator with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`")); - } - assert(column_family); if (read_options.timestamp) { - const Status s = FailIfTsMismatchCf( - column_family, *(read_options.timestamp), /*ts_for_read=*/true); + const Status s = + FailIfTsMismatchCf(column_family, *(read_options.timestamp)); if (!s.ok()) { return NewErrorIterator(s); } @@ -3417,8 +3826,16 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options, ColumnFamilyData* cfd = cfh->cfd(); assert(cfd != nullptr); ReadCallback* read_callback = nullptr; // No read callback provided. + SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + if (read_options.timestamp && read_options.timestamp->size() > 0) { + const Status s = + FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp)); + if (!s.ok()) { + CleanupSuperVersion(sv); + return NewErrorIterator(s); + } + } if (read_options.tailing) { - SuperVersion* sv = cfd->GetReferencedSuperVersion(this); auto iter = new ForwardIterator(this, read_options, cfd, sv, /* allow_unprepared_value */ true); result = NewDBIterator( @@ -3430,7 +3847,7 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options, // Note: no need to consider the special case of // last_seq_same_as_publish_seq_==false since NewIterator is overridden in // WritePreparedTxnDB - result = NewIteratorImpl(read_options, cfd, + result = NewIteratorImpl(read_options, cfd, sv, (read_options.snapshot != nullptr) ? read_options.snapshot->GetSequenceNumber() : kMaxSequenceNumber, @@ -3439,14 +3856,10 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options, return result; } -ArenaWrappedDBIter* DBImpl::NewIteratorImpl(const ReadOptions& read_options, - ColumnFamilyData* cfd, - SequenceNumber snapshot, - ReadCallback* read_callback, - bool expose_blob_index, - bool allow_refresh) { - SuperVersion* sv = cfd->GetReferencedSuperVersion(this); - +ArenaWrappedDBIter* DBImpl::NewIteratorImpl( + const ReadOptions& read_options, ColumnFamilyData* cfd, SuperVersion* sv, + SequenceNumber snapshot, ReadCallback* read_callback, + bool expose_blob_index, bool allow_refresh) { TEST_SYNC_POINT("DBImpl::NewIterator:1"); TEST_SYNC_POINT("DBImpl::NewIterator:2"); @@ -3511,7 +3924,7 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl(const ReadOptions& read_options, env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, sv->current, snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations, sv->version_number, read_callback, this, cfd, expose_blob_index, - read_options.snapshot != nullptr ? false : allow_refresh); + allow_refresh); InternalIterator* internal_iter = NewInternalIterator( db_iter->GetReadOptions(), cfd, sv, db_iter->GetArena(), snapshot, @@ -3522,9 +3935,19 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl(const ReadOptions& read_options, } Status DBImpl::NewIterators( - const ReadOptions& read_options, + const ReadOptions& _read_options, const std::vector& column_families, std::vector* iterators) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { + return Status::InvalidArgument( + "Can only call NewIterators with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; + } if (read_options.managed) { return Status::NotSupported("Managed iterator is not supported anymore."); } @@ -3532,17 +3955,11 @@ Status DBImpl::NewIterators( return Status::NotSupported( "ReadTier::kPersistedData is not yet supported in iterators."); } - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return Status::InvalidArgument( - "Cannot call NewIterators with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - } if (read_options.timestamp) { for (auto* cf : column_families) { assert(cf); - const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp), - /*ts_for_read=*/true); + const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp)); if (!s.ok()) { return s; } @@ -3560,10 +3977,27 @@ Status DBImpl::NewIterators( ReadCallback* read_callback = nullptr; // No read callback provided. iterators->clear(); iterators->reserve(column_families.size()); + autovector> cfd_to_sv; + const bool check_read_ts = + read_options.timestamp && read_options.timestamp->size() > 0; + for (auto cfh : column_families) { + auto cfd = static_cast_with_check(cfh)->cfd(); + SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + cfd_to_sv.emplace_back(cfd, sv); + if (check_read_ts) { + const Status s = + FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp)); + if (!s.ok()) { + for (auto prev_entry : cfd_to_sv) { + CleanupSuperVersion(std::get<1>(prev_entry)); + } + return s; + } + } + } + assert(cfd_to_sv.size() == column_families.size()); if (read_options.tailing) { - for (auto cfh : column_families) { - auto cfd = static_cast_with_check(cfh)->cfd(); - SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + for (auto [cfd, sv] : cfd_to_sv) { auto iter = new ForwardIterator(this, read_options, cfd, sv, /* allow_unprepared_value */ true); iterators->push_back(NewDBIterator( @@ -3579,12 +4013,9 @@ Status DBImpl::NewIterators( auto snapshot = read_options.snapshot != nullptr ? read_options.snapshot->GetSequenceNumber() : versions_->LastSequence(); - for (size_t i = 0; i < column_families.size(); ++i) { - auto* cfd = - static_cast_with_check(column_families[i]) - ->cfd(); + for (auto [cfd, sv] : cfd_to_sv) { iterators->push_back( - NewIteratorImpl(read_options, cfd, snapshot, read_callback)); + NewIteratorImpl(read_options, cfd, sv, snapshot, read_callback)); } } @@ -3812,7 +4243,8 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) { CfdList cf_scheduled; for (auto* cfd : *versions_->GetColumnFamilySet()) { if (!cfd->ioptions()->allow_ingest_behind) { - cfd->current()->storage_info()->UpdateOldestSnapshot(oldest_snapshot); + cfd->current()->storage_info()->UpdateOldestSnapshot( + oldest_snapshot, /*allow_ingest_behind=*/false); if (!cfd->current() ->storage_info() ->BottommostFilesMarkedForCompaction() @@ -3890,7 +4322,6 @@ Status DBImpl::GetPropertiesOfTablesInRange(ColumnFamilyHandle* column_family, return s; } - const std::string& DBImpl::GetName() const { return dbname_; } Env* DBImpl::GetEnv() const { return env_; } @@ -3908,7 +4339,6 @@ SystemClock* DBImpl::GetSystemClock() const { return immutable_db_options_.clock; } - Status DBImpl::StartIOTrace(const TraceOptions& trace_options, std::unique_ptr&& trace_writer) { assert(trace_writer != nullptr); @@ -3921,7 +4351,6 @@ Status DBImpl::EndIOTrace() { return Status::OK(); } - Options DBImpl::GetOptions(ColumnFamilyHandle* column_family) const { InstrumentedMutexLock l(&mutex_); auto cfh = static_cast_with_check(column_family); @@ -4183,9 +4612,19 @@ void DBImpl::GetApproximateMemTableStats(ColumnFamilyHandle* column_family, ColumnFamilyData* cfd = cfh->cfd(); SuperVersion* sv = GetAndRefSuperVersion(cfd); + const Comparator* const ucmp = column_family->GetComparator(); + assert(ucmp); + size_t ts_sz = ucmp->timestamp_size(); + + // Add timestamp if needed + std::string start_with_ts, limit_with_ts; + auto [start, limit] = MaybeAddTimestampsToRange( + &range.start, &range.limit, ts_sz, &start_with_ts, &limit_with_ts); + assert(start.has_value()); + assert(limit.has_value()); // Convert user_key into a corresponding internal key. - InternalKey k1(range.start, kMaxSequenceNumber, kValueTypeForSeek); - InternalKey k2(range.limit, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k1(start.value(), kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k2(limit.value(), kMaxSequenceNumber, kValueTypeForSeek); MemTable::MemTableStats memStats = sv->mem->ApproximateStats(k1.Encode(), k2.Encode()); MemTable::MemTableStats immStats = @@ -4216,23 +4655,16 @@ Status DBImpl::GetApproximateSizes(const SizeApproximationOptions& options, // TODO: plumb Env::IOActivity const ReadOptions read_options; for (int i = 0; i < n; i++) { - Slice start = range[i].start; - Slice limit = range[i].limit; - // Add timestamp if needed std::string start_with_ts, limit_with_ts; - if (ts_sz > 0) { - // Maximum timestamp means including all key with any timestamp - AppendKeyWithMaxTimestamp(&start_with_ts, start, ts_sz); - // Append a maximum timestamp as the range limit is exclusive: - // [start, limit) - AppendKeyWithMaxTimestamp(&limit_with_ts, limit, ts_sz); - start = start_with_ts; - limit = limit_with_ts; - } + auto [start, limit] = + MaybeAddTimestampsToRange(&range[i].start, &range[i].limit, ts_sz, + &start_with_ts, &limit_with_ts); + assert(start.has_value()); + assert(limit.has_value()); // Convert user_key into a corresponding internal key. - InternalKey k1(start, kMaxSequenceNumber, kValueTypeForSeek); - InternalKey k2(limit, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k1(start.value(), kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k2(limit.value(), kMaxSequenceNumber, kValueTypeForSeek); sizes[i] = 0; if (options.include_files) { sizes[i] += versions_->ApproximateSize( @@ -4445,10 +4877,12 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family, deleted_files.insert(level_file); level_file->being_compacted = true; } - vstorage->ComputeCompactionScore(*cfd->ioptions(), - *cfd->GetLatestMutableCFOptions()); } } + if (!deleted_files.empty()) { + vstorage->ComputeCompactionScore(*cfd->ioptions(), + *cfd->GetLatestMutableCFOptions()); + } if (edit.GetDeletedFiles().empty()) { job_context.Clean(); return status; @@ -4869,16 +5303,13 @@ Status DestroyDB(const std::string& dbname, const Options& options, return result; } -Status DBImpl::WriteOptionsFile(bool need_mutex_lock, - bool need_enter_write_thread) { - WriteThread::Writer w; - if (need_mutex_lock) { - mutex_.Lock(); - } else { +Status DBImpl::WriteOptionsFile(bool db_mutex_already_held) { + options_mutex_.AssertHeld(); + + if (db_mutex_already_held) { mutex_.AssertHeld(); - } - if (need_enter_write_thread) { - write_thread_.EnterUnbatched(&w, &mutex_); + } else { + mutex_.Lock(); } std::vector cf_names; @@ -4893,10 +5324,10 @@ Status DBImpl::WriteOptionsFile(bool need_mutex_lock, cf_opts.push_back(cfd->GetLatestCFOptions()); } - // Unlock during expensive operations. New writes cannot get here - // because the single write thread ensures all new writes get queued. DBOptions db_options = BuildDBOptions(immutable_db_options_, mutable_db_options_); + + // Unlock during expensive operations. mutex_.Unlock(); TEST_SYNC_POINT("DBImpl::WriteOptionsFile:1"); @@ -4921,22 +5352,22 @@ Status DBImpl::WriteOptionsFile(bool need_mutex_lock, } } - // restore lock - if (!need_mutex_lock) { - mutex_.Lock(); - } - if (need_enter_write_thread) { - write_thread_.ExitUnbatched(&w); - } if (!s.ok()) { ROCKS_LOG_WARN(immutable_db_options_.info_log, "Unnable to persist options -- %s", s.ToString().c_str()); if (immutable_db_options_.fail_if_options_file_error) { - return Status::IOError("Unable to persist options.", - s.ToString().c_str()); + s = Status::IOError("Unable to persist options.", s.ToString().c_str()); + } else { + // Ignore error + s = Status::OK(); } } - return Status::OK(); + + // Restore lock if appropriate + if (db_mutex_already_held) { + mutex_.Lock(); + } + return s; } namespace { @@ -5763,7 +6194,7 @@ Status DBImpl::ClipColumnFamily(ColumnFamilyHandle* column_family, if (status.ok()) { // Delete [clip_end_key, largest_use_key] - if (ucmp->Compare(end_key, largest_user_key) < 0) { + if (ucmp->Compare(end_key, largest_user_key) <= 0) { status = DeleteRange(wo, column_family, end_key, largest_user_key); if (status.ok()) { status = Delete(wo, column_family, largest_user_key); @@ -5784,17 +6215,40 @@ Status DBImpl::ClipColumnFamily(ColumnFamilyHandle* column_family, // last level to compact to and that range tombstones are not dropped // during non-bottommost compactions, calling CompactRange() on these two // ranges may not clear all range tombstones. - status = CompactRange(compact_options, nullptr, nullptr); + status = CompactRange(compact_options, column_family, nullptr, nullptr); } return status; } -Status DBImpl::VerifyFileChecksums(const ReadOptions& read_options) { - return VerifyChecksumInternal(read_options, /*use_file_checksum=*/true); +Status DBImpl::VerifyFileChecksums(const ReadOptions& _read_options) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kVerifyFileChecksums) { + return Status::InvalidArgument( + "Can only call VerifyFileChecksums with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or " + "`Env::IOActivity::kVerifyFileChecksums`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kVerifyFileChecksums; + } + return VerifyChecksumInternal(read_options, + /*use_file_checksum=*/true); } -Status DBImpl::VerifyChecksum(const ReadOptions& read_options) { - return VerifyChecksumInternal(read_options, /*use_file_checksum=*/false); +Status DBImpl::VerifyChecksum(const ReadOptions& _read_options) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kVerifyDBChecksum) { + return Status::InvalidArgument( + "Can only call VerifyChecksum with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kVerifyDBChecksum`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kVerifyDBChecksum; + } + return VerifyChecksumInternal(read_options, + /*use_file_checksum=*/false); } Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options, @@ -5806,12 +6260,6 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options, Status s; - if (read_options.io_activity != Env::IOActivity::kUnknown) { - s = Status::InvalidArgument( - "Cannot verify file checksum with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - return s; - } if (use_file_checksum) { FileChecksumGenFactory* const file_checksum_gen_factory = immutable_db_options_.file_checksum_gen_factory.get(); @@ -5822,6 +6270,7 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options, return s; } } + // FIXME? What does it mean if read_options.verify_checksums == false? // TODO: simplify using GetRefedColumnFamilySet? std::vector cfd_list; @@ -5862,7 +6311,7 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options, fmeta->file_checksum_func_name, fname, read_options); } else { - s = ROCKSDB_NAMESPACE::VerifySstFileChecksum( + s = ROCKSDB_NAMESPACE::VerifySstFileChecksumInternal( opts, file_options_, read_options, fname, fd.largest_seqno); } RecordTick(stats_, VERIFY_CHECKSUM_READ_BYTES, @@ -5925,12 +6374,6 @@ Status DBImpl::VerifyFullFileChecksum(const std::string& file_checksum_expected, const std::string& func_name_expected, const std::string& fname, const ReadOptions& read_options) { - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return Status::InvalidArgument( - "Cannot call VerifyChecksum with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - } - Status s; if (file_checksum_expected == kUnknownFileChecksum) { return s; @@ -5941,8 +6384,8 @@ Status DBImpl::VerifyFullFileChecksum(const std::string& file_checksum_expected, fs_.get(), fname, immutable_db_options_.file_checksum_gen_factory.get(), func_name_expected, &file_checksum, &func_name, read_options.readahead_size, immutable_db_options_.allow_mmap_reads, - io_tracer_, immutable_db_options_.rate_limiter.get(), - read_options.rate_limiter_priority); + io_tracer_, immutable_db_options_.rate_limiter.get(), read_options, + immutable_db_options_.stats, immutable_db_options_.clock); if (s.ok()) { assert(func_name_expected == func_name); if (file_checksum != file_checksum_expected) { @@ -6119,19 +6562,51 @@ Status DBImpl::GetCreationTimeOfOldestFile(uint64_t* creation_time) { } } -void DBImpl::RecordSeqnoToTimeMapping() { - // Get time first then sequence number, so the actual time of seqno is <= - // unix_time recorded - int64_t unix_time = 0; - immutable_db_options_.clock->GetCurrentTime(&unix_time) - .PermitUncheckedError(); // Ignore error +void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) { + // TECHNICALITY: Sample last sequence number *before* time, as prescribed + // for SeqnoToTimeMapping. We don't know how long it has been since the last + // sequence number was written, so we at least have a one-sided bound by + // sampling in this order. SequenceNumber seqno = GetLatestSequenceNumber(); + int64_t unix_time_signed = 0; + immutable_db_options_.clock->GetCurrentTime(&unix_time_signed) + .PermitUncheckedError(); // Ignore error + uint64_t unix_time = static_cast(unix_time_signed); bool appended = false; { InstrumentedMutexLock l(&mutex_); - appended = seqno_time_mapping_.Append(seqno, unix_time); + if (populate_historical_seconds > 0) { + if (seqno > 1 && unix_time > populate_historical_seconds) { + // seqno=0 is reserved + SequenceNumber from_seqno = 1; + appended = seqno_to_time_mapping_.PrePopulate( + from_seqno, seqno, unix_time - populate_historical_seconds, + unix_time); + } else { + // One of these will fail + assert(seqno > 1); + assert(unix_time > populate_historical_seconds); + } + } else { + // FIXME: assert(seqno > 0); + appended = seqno_to_time_mapping_.Append(seqno, unix_time); + } } - if (!appended) { + if (populate_historical_seconds > 0) { + if (appended) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "Pre-populated sequence number to time entries: [1,%" PRIu64 + "] -> [%" PRIu64 ",%" PRIu64 "]", + seqno, unix_time - populate_historical_seconds, unix_time); + } else { + ROCKS_LOG_WARN( + immutable_db_options_.info_log, + "Failed to pre-populate sequence number to time entries: [1,%" PRIu64 + "] -> [%" PRIu64 ",%" PRIu64 "]", + seqno, unix_time - populate_historical_seconds, unix_time); + } + } else if (!appended) { ROCKS_LOG_WARN(immutable_db_options_.info_log, "Failed to insert sequence number to time entry: %" PRIu64 " -> %" PRIu64, diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index 57c43b517..34a5f3398 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -197,6 +197,8 @@ class DBImpl : public DB { Status PutEntity(const WriteOptions& options, ColumnFamilyHandle* column_family, const Slice& key, const WideColumns& columns) override; + Status PutEntity(const WriteOptions& options, const Slice& key, + const AttributeGroups& attribute_groups) override; using DB::Merge; Status Merge(const WriteOptions& options, ColumnFamilyHandle* column_family, @@ -234,7 +236,7 @@ class DBImpl : public DB { virtual Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) override; - virtual Status Get(const ReadOptions& options, + virtual Status Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value, std::string* timestamp) override; @@ -242,6 +244,8 @@ class DBImpl : public DB { Status GetEntity(const ReadOptions& options, ColumnFamilyHandle* column_family, const Slice& key, PinnableWideColumns* columns) override; + Status GetEntity(const ReadOptions& options, const Slice& key, + PinnableAttributeGroups* result) override; using DB::GetMergeOperands; Status GetMergeOperands(const ReadOptions& options, @@ -265,7 +269,7 @@ class DBImpl : public DB { const std::vector& keys, std::vector* values) override; virtual std::vector MultiGet( - const ReadOptions& options, + const ReadOptions& _read_options, const std::vector& column_family, const std::vector& keys, std::vector* values, std::vector* timestamps) override; @@ -280,8 +284,9 @@ class DBImpl : public DB { void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, const bool sorted_input = false) override; - void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family, - const size_t num_keys, const Slice* keys, PinnableSlice* values, + void MultiGet(const ReadOptions& _read_options, + ColumnFamilyHandle* column_family, const size_t num_keys, + const Slice* keys, PinnableSlice* values, std::string* timestamps, Status* statuses, const bool sorted_input = false) override; @@ -289,13 +294,13 @@ class DBImpl : public DB { ColumnFamilyHandle** column_families, const Slice* keys, PinnableSlice* values, Status* statuses, const bool sorted_input = false) override; - void MultiGet(const ReadOptions& options, const size_t num_keys, + void MultiGet(const ReadOptions& _read_options, const size_t num_keys, ColumnFamilyHandle** column_families, const Slice* keys, PinnableSlice* values, std::string* timestamps, Status* statuses, const bool sorted_input = false) override; void MultiGetWithCallback( - const ReadOptions& options, ColumnFamilyHandle* column_family, + const ReadOptions& _read_options, ColumnFamilyHandle* column_family, ReadCallback* callback, autovector* sorted_keys); @@ -310,6 +315,9 @@ class DBImpl : public DB { ColumnFamilyHandle** column_families, const Slice* keys, PinnableWideColumns* results, Status* statuses, bool sorted_input) override; + void MultiGetEntity(const ReadOptions& options, size_t num_keys, + const Slice* keys, + PinnableAttributeGroups* results) override; virtual Status CreateColumnFamily(const ColumnFamilyOptions& cf_options, const std::string& column_family, @@ -336,10 +344,10 @@ class DBImpl : public DB { bool* value_found = nullptr) override; using DB::NewIterator; - virtual Iterator* NewIterator(const ReadOptions& options, + virtual Iterator* NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) override; virtual Status NewIterators( - const ReadOptions& options, + const ReadOptions& _read_options, const std::vector& column_families, std::vector* iterators) override; @@ -627,6 +635,14 @@ class DBImpl : public DB { int* number_of_operands = nullptr; }; + Status GetImpl(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value); + + Status GetImpl(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, std::string* timestamp); + // Function that Get and KeyMayExist call with no_io true or false // Note: 'value_found' from KeyMayExist propagates here // This function is also called by GetMergeOperands @@ -634,12 +650,12 @@ class DBImpl : public DB { // get_impl_options.key via get_impl_options.value // If get_impl_options.get_value = false get merge operands associated with // get_impl_options.key via get_impl_options.merge_operands - Status GetImpl(const ReadOptions& options, const Slice& key, - GetImplOptions& get_impl_options); + virtual Status GetImpl(const ReadOptions& options, const Slice& key, + GetImplOptions& get_impl_options); // If `snapshot` == kMaxSequenceNumber, set a recent one inside the file. ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options, - ColumnFamilyData* cfd, + ColumnFamilyData* cfd, SuperVersion* sv, SequenceNumber snapshot, ReadCallback* read_callback, bool expose_blob_index = false, @@ -1174,6 +1190,7 @@ class DBImpl : public DB { size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const; void TEST_WaitForPeriodicTaskRun(std::function callback) const; SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const; + const autovector& TEST_GetFilesToQuarantine() const; size_t TEST_EstimateInMemoryStatsHistorySize() const; uint64_t TEST_GetCurrentLogNumber() const { @@ -1188,6 +1205,10 @@ class DBImpl : public DB { const PeriodicTaskScheduler& TEST_GetPeriodicTaskScheduler() const; + static Status TEST_ValidateOptions(const DBOptions& db_options) { + return ValidateOptions(db_options); + } + #endif // NDEBUG // persist stats to column family "_persistent_stats" @@ -1199,8 +1220,11 @@ class DBImpl : public DB { // flush LOG out of application buffer void FlushInfoLog(); - // record current sequence number to time mapping - void RecordSeqnoToTimeMapping(); + // record current sequence number to time mapping. If + // populate_historical_seconds > 0 then pre-populate all the + // sequence numbers from [1, last] to map to [now minus + // populate_historical_seconds, now]. + void RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds); // Interface to block and signal the DB in case of stalling writes by // WriteBufferManager. Each DBImpl object contains ptr to WBMStallInterface. @@ -1340,6 +1364,11 @@ class DBImpl : public DB { std::atomic shutting_down_; + // No new background jobs can be queued if true. This is used to prevent new + // background jobs from being queued after WaitForCompact() completes waiting + // all background jobs then attempts to close when close_db_ option is true. + bool reject_new_background_jobs_; + // RecoveryContext struct stores the context about version edits along // with corresponding column_family_data and column_family_options. class RecoveryContext { @@ -1369,15 +1398,15 @@ class DBImpl : public DB { autovector cfds_; autovector mutable_cf_opts_; autovector> edit_lists_; - // files_to_delete_ contains sst files - std::unordered_set files_to_delete_; + // Stale SST files to delete found upon recovery. This stores a mapping from + // such a file's absolute path to its parent directory. + std::unordered_map files_to_delete_; + bool is_new_db_ = false; }; - // Except in DB::Open(), WriteOptionsFile can only be called when: - // Persist options to options file. - // If need_mutex_lock = false, the method will lock DB mutex. - // If need_enter_write_thread = false, the method will enter write thread. - Status WriteOptionsFile(bool need_mutex_lock, bool need_enter_write_thread); + // Persist options to options file. Must be holding options_mutex_. + // Will lock DB mutex if !db_mutex_already_held. + Status WriteOptionsFile(bool db_mutex_already_held); Status CompactRangeInternal(const CompactRangeOptions& options, ColumnFamilyHandle* column_family, @@ -1529,8 +1558,18 @@ class DBImpl : public DB { void SetDbSessionId(); Status FailIfCfHasTs(const ColumnFamilyHandle* column_family) const; - Status FailIfTsMismatchCf(ColumnFamilyHandle* column_family, const Slice& ts, - bool ts_for_read) const; + Status FailIfTsMismatchCf(ColumnFamilyHandle* column_family, + const Slice& ts) const; + + // Check that the read timestamp `ts` is at or above the `full_history_ts_low` + // timestamp in a `SuperVersion`. It's necessary to do this check after + // grabbing the SuperVersion. If the check passed, the referenced SuperVersion + // this read holds on to can ensure the read won't be affected if + // `full_history_ts_low` is increased concurrently, and this achieves that + // without explicitly locking by piggybacking the SuperVersion. + Status FailIfReadCollapsedHistory(const ColumnFamilyData* cfd, + const SuperVersion* sv, + const Slice& ts) const; // recovery_ctx stores the context about version edits and // LogAndApplyForRecovery persist all those edits to new Manifest after @@ -1566,12 +1605,14 @@ class DBImpl : public DB { friend class ForwardIterator; friend struct SuperVersion; friend class CompactedDBImpl; +#ifndef NDEBUG friend class DBTest_ConcurrentFlushWAL_Test; friend class DBTest_MixedSlowdownOptionsStop_Test; friend class DBCompactionTest_CompactBottomLevelFilesWithDeletions_Test; friend class DBCompactionTest_CompactionDuringShutdown_Test; + friend class DBCompactionTest_DelayCompactBottomLevelFilesWithDeletions_Test; + friend class DBCompactionTest_DisableCompactBottomLevelFiles_Test; friend class StatsHistoryTest_PersistentStatsCreateColumnFamilies_Test; -#ifndef NDEBUG friend class DBTest2_ReadCallbackTest_Test; friend class WriteCallbackPTest_WriteWithCallbackTest_Test; friend class XFTransactionWriteHandler; @@ -1793,10 +1834,15 @@ class DBImpl : public DB { const Status CreateArchivalDirectory(); + // Create a column family, without some of the follow-up work yet Status CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options, const std::string& cf_name, ColumnFamilyHandle** handle); + // Follow-up work to user creating a column family or (families) + Status WrapUpCreateColumnFamilies( + const std::vector& cf_options); + Status DropColumnFamilyImpl(ColumnFamilyHandle* column_family); // Delete any unneeded files and stale in-memory entries. @@ -1826,7 +1872,8 @@ class DBImpl : public DB { void ReleaseFileNumberFromPendingOutputs( std::unique_ptr::iterator>& v); - IOStatus SyncClosedLogs(JobContext* job_context, VersionEdit* synced_wals); + IOStatus SyncClosedLogs(JobContext* job_context, VersionEdit* synced_wals, + bool error_recovery_in_prog); // Flush the in-memory write buffer to storage. Switches to a new // log-file/memtable and writes a new descriptor iff successful. Then @@ -1921,6 +1968,8 @@ class DBImpl : public DB { const autovector& provided_candidate_cfds = {}, bool entered_write_thread = false); + Status RetryFlushesForErrorRecovery(FlushReason flush_reason, bool wait); + // Wait until flushing this column family won't stall writes Status WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd, bool* flush_needed); @@ -2063,8 +2112,18 @@ class DBImpl : public DB { // flush is considered complete. std::unordered_map cfd_to_max_mem_id_to_persist; + +#ifndef NDEBUG + int reschedule_count = 1; +#endif /* !NDEBUG */ }; + // In case of atomic flush, generates a `FlushRequest` for the latest atomic + // cuts for these `cfds`. Atomic cuts are recorded in + // `AssignAtomicFlushSeq()`. For each entry in `cfds`, all CFDs sharing the + // same latest atomic cut must also be present. + // + // REQUIRES: mutex held void GenerateFlushRequest(const autovector& cfds, FlushReason flush_reason, FlushRequest* req); @@ -2091,6 +2150,7 @@ class DBImpl : public DB { Env::Priority thread_pri); Status BackgroundFlush(bool* madeProgress, JobContext* job_context, LogBuffer* log_buffer, FlushReason* reason, + bool* flush_rescheduled_to_retain_udt, Env::Priority thread_pri); bool EnoughRoomForCompaction(ColumnFamilyData* cfd, @@ -2103,10 +2163,19 @@ class DBImpl : public DB { std::unique_ptr* token, LogBuffer* log_buffer); + // Return true if the `FlushRequest` can be rescheduled to retain the UDT. + // Only true if there are user-defined timestamps in the involved MemTables + // with newer than cutoff timestamp `full_history_ts_low` and not flushing + // immediately will not cause entering write stall mode. + bool ShouldRescheduleFlushRequestToRetainUDT(const FlushRequest& flush_req); + // Schedule background tasks Status StartPeriodicTaskScheduler(); - Status RegisterRecordSeqnoTimeWorker(); + // Cancel scheduled periodic tasks + Status CancelPeriodicTaskScheduler(); + + Status RegisterRecordSeqnoTimeWorker(bool is_new_db); void PrintStatistics(); @@ -2186,10 +2255,11 @@ class DBImpl : public DB { bool ShouldntRunManualCompaction(ManualCompactionState* m); bool HaveManualCompaction(ColumnFamilyData* cfd); bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1); + void UpdateDeletionCompactionStats(const std::unique_ptr& c); void BuildCompactionJobInfo(const ColumnFamilyData* cfd, Compaction* c, const Status& st, const CompactionJobStats& compaction_job_stats, - const int job_id, const Version* current, + const int job_id, CompactionJobInfo* compaction_job_info) const; // Reserve the next 'num' file numbers for to-be-ingested external SST files, // and return the current file_number in 'next_file_number'. @@ -2282,15 +2352,18 @@ class DBImpl : public DB { // If callback is non-null, the callback is refreshed with the snapshot // sequence number // - // A return value of true indicates that the SuperVersions were obtained - // from the ColumnFamilyData, whereas false indicates they are thread - // local + // `sv_from_thread_local` being set to false indicates that the SuperVersion + // obtained from the ColumnFamilyData, whereas true indicates they are thread + // local. + // A non-OK status will be returned if for a column family that enables + // user-defined timestamp feature, the specified `ReadOptions.timestamp` + // attemps to read collapsed history. template - bool MultiCFSnapshot( + Status MultiCFSnapshot( const ReadOptions& read_options, ReadCallback* callback, std::function& iter_deref_func, - T* cf_list, SequenceNumber* snapshot); + T* cf_list, SequenceNumber* snapshot, bool* sv_from_thread_local); // The actual implementation of the batching MultiGet. The caller is expected // to have acquired the SuperVersion and pass in a snapshot sequence number @@ -2301,6 +2374,11 @@ class DBImpl : public DB { autovector* sorted_keys, SuperVersion* sv, SequenceNumber snap_seqnum, ReadCallback* callback); + void MultiGetWithCallbackImpl( + const ReadOptions& read_options, ColumnFamilyHandle* column_family, + ReadCallback* callback, + autovector* sorted_keys); + Status DisableFileDeletionsWithLock(); Status IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd, @@ -2311,9 +2389,19 @@ class DBImpl : public DB { // Lock over the persistent DB state. Non-nullptr iff successfully acquired. FileLock* db_lock_; - // In addition to mutex_, log_write_mutex_ protected writes to stats_history_ + // Guards changes to DB and CF options to ensure consistency between + // * In-memory options objects + // * Settings in effect + // * Options file contents + // while allowing the DB mutex to be released during slow operations like + // persisting options file or modifying global periodic task timer. + // Always acquired *before* DB mutex when this one is applicable. + InstrumentedMutex options_mutex_; + + // Guards reads and writes to in-memory stats_history_. InstrumentedMutex stats_history_mutex_; - // In addition to mutex_, log_write_mutex_ protected writes to logs_ and + + // In addition to mutex_, log_write_mutex_ protects writes to logs_ and // logfile_number_. With two_write_queues it also protects alive_log_files_, // and log_empty_. Refer to the definition of each variable below for more // details. @@ -2590,9 +2678,6 @@ class DBImpl : public DB { // initialized with startup time. uint64_t delete_obsolete_files_last_run_; - // last time stats were dumped to LOG - std::atomic last_stats_dump_time_microsec_; - // The thread that wants to switch memtable, can wait on this cv until the // pending writes to memtable finishes. std::condition_variable switch_cv_; @@ -2703,9 +2788,9 @@ class DBImpl : public DB { // Pointer to WriteBufferManager stalling interface. std::unique_ptr wbm_stall_; - // seqno_time_mapping_ stores the sequence number to time mapping, it's not + // seqno_to_time_mapping_ stores the sequence number to time mapping, it's not // thread safe, both read and write need db mutex hold. - SeqnoToTimeMapping seqno_time_mapping_; + SeqnoToTimeMapping seqno_to_time_mapping_; // Stop write token that is acquired when first LockWAL() is called. // Destroyed when last UnlockWAL() is called. Controlled by DB mutex. @@ -2785,7 +2870,9 @@ static void ClipToRange(T* ptr, V minvalue, V maxvalue) { inline Status DBImpl::FailIfCfHasTs( const ColumnFamilyHandle* column_family) const { - column_family = column_family ? column_family : DefaultColumnFamily(); + if (!column_family) { + return Status::InvalidArgument("column family handle cannot be null"); + } assert(column_family); const Comparator* const ucmp = column_family->GetComparator(); assert(ucmp); @@ -2799,8 +2886,7 @@ inline Status DBImpl::FailIfCfHasTs( } inline Status DBImpl::FailIfTsMismatchCf(ColumnFamilyHandle* column_family, - const Slice& ts, - bool ts_for_read) const { + const Slice& ts) const { if (!column_family) { return Status::InvalidArgument("column family handle cannot be null"); } @@ -2820,20 +2906,28 @@ inline Status DBImpl::FailIfTsMismatchCf(ColumnFamilyHandle* column_family, << ts_sz << " given"; return Status::InvalidArgument(oss.str()); } - if (ts_for_read) { - auto cfh = static_cast_with_check(column_family); - auto cfd = cfh->cfd(); - std::string current_ts_low = cfd->GetFullHistoryTsLow(); - if (!current_ts_low.empty() && - ucmp->CompareTimestamp(ts, current_ts_low) < 0) { - std::stringstream oss; - oss << "Read timestamp: " << ts.ToString(true) - << " is smaller than full_history_ts_low: " - << Slice(current_ts_low).ToString(true) << std::endl; - return Status::InvalidArgument(oss.str()); - } - } return Status::OK(); } +inline Status DBImpl::FailIfReadCollapsedHistory(const ColumnFamilyData* cfd, + const SuperVersion* sv, + const Slice& ts) const { + // Reaching to this point means the timestamp size matching sanity check in + // `DBImpl::FailIfTsMismatchCf` already passed. So we skip that and assume + // column family has the same user-defined timestamp format as `ts`. + const Comparator* const ucmp = cfd->user_comparator(); + assert(ucmp); + const std::string& full_history_ts_low = sv->full_history_ts_low; + assert(full_history_ts_low.empty() || + full_history_ts_low.size() == ts.size()); + if (!full_history_ts_low.empty() && + ucmp->CompareTimestamp(ts, full_history_ts_low) < 0) { + std::stringstream oss; + oss << "Read timestamp: " << ts.ToString(true) + << " is smaller than full_history_ts_low: " + << Slice(full_history_ts_low).ToString(true) << std::endl; + return Status::InvalidArgument(oss.str()); + } + return Status::OK(); +} } // namespace ROCKSDB_NAMESPACE diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index c64d4ecdb..71c23de95 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -21,7 +21,9 @@ #include "monitoring/thread_status_util.h" #include "test_util/sync_point.h" #include "util/cast_util.h" +#include "util/coding.h" #include "util/concurrent_task_limiter_impl.h" +#include "util/udt_util.h" namespace ROCKSDB_NAMESPACE { @@ -76,8 +78,43 @@ bool DBImpl::RequestCompactionToken(ColumnFamilyData* cfd, bool force, return false; } +bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT( + const FlushRequest& flush_req) { + mutex_.AssertHeld(); + assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1); + ColumnFamilyData* cfd = flush_req.cfd_to_max_mem_id_to_persist.begin()->first; + uint64_t max_memtable_id = + flush_req.cfd_to_max_mem_id_to_persist.begin()->second; + if (cfd->IsDropped() || + !cfd->ShouldPostponeFlushToRetainUDT(max_memtable_id)) { + return false; + } + // Check if holding on the flush will cause entering write stall mode. + // Write stall entered because of the accumulation of write buffers can be + // alleviated if we continue with the flush instead of postponing it. + const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions(); + + // Taking the status of the active Memtable into consideration so that we are + // not just checking if DB is currently already in write stall mode. + int mem_to_flush = cfd->mem()->ApproximateMemoryUsageFast() >= + cfd->mem()->write_buffer_size() / 2 + ? 1 + : 0; + WriteStallCondition write_stall = + ColumnFamilyData::GetWriteStallConditionAndCause( + cfd->imm()->NumNotFlushed() + mem_to_flush, /*num_l0_files=*/0, + /*num_compaction_needed_bytes=*/0, mutable_cf_options, + *cfd->ioptions()) + .first; + if (write_stall != WriteStallCondition::kNormal) { + return false; + } + return true; +} + IOStatus DBImpl::SyncClosedLogs(JobContext* job_context, - VersionEdit* synced_wals) { + VersionEdit* synced_wals, + bool error_recovery_in_prog) { TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Start"); InstrumentedMutexLock l(&log_write_mutex_); autovector logs_to_sync; @@ -103,7 +140,7 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context, ROCKS_LOG_INFO(immutable_db_options_.info_log, "[JOB %d] Syncing log #%" PRIu64, job_context->job_id, log->get_log_number()); - if (error_handler_.IsRecoveryInProgress()) { + if (error_recovery_in_prog) { log->file()->reset_seen_error(); } io_s = log->file()->Sync(immutable_db_options_.use_fsync); @@ -112,7 +149,7 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context, } if (immutable_db_options_.recycle_log_file_num > 0) { - if (error_handler_.IsRecoveryInProgress()) { + if (error_recovery_in_prog) { log->file()->reset_seen_error(); } io_s = log->Close(); @@ -186,9 +223,10 @@ Status DBImpl::FlushMemTableToOutputFile( // `snapshot_seqs` has already been computed before this function starts. // Recording the max memtable ID ensures that the flush job does not flush // a memtable without knowing such snapshot(s). - uint64_t max_memtable_id = needs_to_sync_closed_wals - ? cfd->imm()->GetLatestMemTableID() - : std::numeric_limits::max(); + uint64_t max_memtable_id = + needs_to_sync_closed_wals + ? cfd->imm()->GetLatestMemTableID(false /* for_atomic_flush */) + : std::numeric_limits::max(); // If needs_to_sync_closed_wals is false, then the flush job will pick ALL // existing memtables of the column family when PickMemTable() is called @@ -197,7 +235,7 @@ Status DBImpl::FlushMemTableToOutputFile( // releases and re-acquires the db mutex. In the meantime, the application // can still insert into the memtables and increase the db's sequence number. // The application can take a snapshot, hoping that the latest visible state - // to this snapshto is preserved. This is hard to guarantee since db mutex + // to this snapshot is preserved. This is hard to guarantee since db mutex // not held. This newly-created snapshot is not included in `snapshot_seqs` // and the flush job is unaware of its presence. Consequently, the flush job // may drop certain keys when generating the L0, causing incorrect data to be @@ -214,7 +252,7 @@ Status DBImpl::FlushMemTableToOutputFile( GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_, &event_logger_, mutable_cf_options.report_bg_io_stats, true /* sync_output_directory */, true /* write_manifest */, thread_pri, - io_tracer_, seqno_time_mapping_, db_id_, db_session_id_, + io_tracer_, seqno_to_time_mapping_, db_id_, db_session_id_, cfd->GetFullHistoryTsLow(), &blob_callback_); FileMetaData file_meta; @@ -225,8 +263,10 @@ Status DBImpl::FlushMemTableToOutputFile( // SyncClosedLogs() may unlock and re-lock the log_write_mutex multiple // times. VersionEdit synced_wals; + bool error_recovery_in_prog = error_handler_.IsRecoveryInProgress(); mutex_.Unlock(); - log_io_s = SyncClosedLogs(job_context, &synced_wals); + log_io_s = + SyncClosedLogs(job_context, &synced_wals, error_recovery_in_prog); mutex_.Lock(); if (log_io_s.ok() && synced_wals.IsWalAddition()) { const ReadOptions read_options(Env::IOActivity::kFlush); @@ -248,6 +288,24 @@ Status DBImpl::FlushMemTableToOutputFile( // If the log sync failed, we do not need to pick memtable. Otherwise, // num_flush_not_started_ needs to be rollback. TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforePickMemtables"); + // Exit a flush due to bg error should not set bg error again. + bool skip_set_bg_error = false; + if (s.ok() && !error_handler_.GetBGError().ok() && + error_handler_.IsBGWorkStopped() && + flush_reason != FlushReason::kErrorRecovery && + flush_reason != FlushReason::kErrorRecoveryRetryFlush) { + // Error recovery in progress, should not pick memtable which excludes + // them from being picked up by recovery flush. + // This ensures that when bg error is set, no new flush can pick + // memtables. + skip_set_bg_error = true; + s = error_handler_.GetBGError(); + assert(!s.ok()); + ROCKS_LOG_BUFFER(log_buffer, + "[JOB %d] Skip flush due to background error %s", + job_context->job_id, s.ToString().c_str()); + } + if (s.ok()) { flush_job.PickMemTable(); need_cancel = true; @@ -268,7 +326,8 @@ Status DBImpl::FlushMemTableToOutputFile( // is unlocked by the current thread. if (s.ok()) { s = flush_job.Run(&logs_with_prep_tracker_, &file_meta, - &switched_to_mempurge); + &switched_to_mempurge, &skip_set_bg_error, + &error_handler_); need_cancel = false; } @@ -309,7 +368,8 @@ Status DBImpl::FlushMemTableToOutputFile( } } - if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped()) { + if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped() && + !skip_set_bg_error) { if (log_io_s.ok()) { // Error while writing to MANIFEST. // In fact, versions_->io_status() can also be the result of renaming @@ -466,7 +526,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_, &event_logger_, mutable_cf_options.report_bg_io_stats, false /* sync_output_directory */, false /* write_manifest */, - thread_pri, io_tracer_, seqno_time_mapping_, db_id_, db_session_id_, + thread_pri, io_tracer_, seqno_to_time_mapping_, db_id_, db_session_id_, cfd->GetFullHistoryTsLow(), &blob_callback_)); } @@ -490,8 +550,10 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( // TODO (yanqin) investigate whether we should sync the closed logs for // single column family case. VersionEdit synced_wals; + bool error_recovery_in_prog = error_handler_.IsRecoveryInProgress(); mutex_.Unlock(); - log_io_s = SyncClosedLogs(job_context, &synced_wals); + log_io_s = + SyncClosedLogs(job_context, &synced_wals, error_recovery_in_prog); mutex_.Lock(); if (log_io_s.ok() && synced_wals.IsWalAddition()) { const ReadOptions read_options(Env::IOActivity::kFlush); @@ -521,6 +583,21 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( pick_status.push_back(false); } + bool flush_for_recovery = + bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery || + bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecoveryRetryFlush; + bool skip_set_bg_error = false; + + if (s.ok() && !error_handler_.GetBGError().ok() && + error_handler_.IsBGWorkStopped() && !flush_for_recovery) { + s = error_handler_.GetBGError(); + skip_set_bg_error = true; + assert(!s.ok()); + ROCKS_LOG_BUFFER(log_buffer, + "[JOB %d] Skip flush due to background error %s", + job_context->job_id, s.ToString().c_str()); + } + if (s.ok()) { for (int i = 0; i != num_cfs; ++i) { jobs[i]->PickMemTable(); @@ -585,7 +662,10 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( } } } - } else { + } else if (!skip_set_bg_error) { + // When `skip_set_bg_error` is true, no memtable is picked so + // there is no need to call Cancel() or RollbackMemtableFlush(). + // // Need to undo atomic flush if something went wrong, i.e. s is not OK and // it is not because of CF drop. // Have to cancel the flush jobs that have NOT executed because we need to @@ -598,8 +678,8 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( for (int i = 0; i != num_cfs; ++i) { if (exec_status[i].second.ok() && exec_status[i].first) { auto& mems = jobs[i]->GetMemTables(); - cfds[i]->imm()->RollbackMemtableFlush(mems, - file_meta[i].fd.GetNumber()); + cfds[i]->imm()->RollbackMemtableFlush( + mems, /*rollback_succeeding_memtables=*/false); } } } @@ -641,10 +721,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( }; bool resuming_from_bg_err = - error_handler_.IsDBStopped() || - (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery || - bg_flush_args[0].flush_reason_ == - FlushReason::kErrorRecoveryRetryFlush); + error_handler_.IsDBStopped() || flush_for_recovery; while ((!resuming_from_bg_err || error_handler_.GetRecoveryError().ok())) { std::pair res = wait_to_install_func(); @@ -655,15 +732,27 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( s = res.first; break; } else if (!res.second) { + // we are the oldest immutable memtable + break; + } + // We are not the oldest immutable memtable + TEST_SYNC_POINT_CALLBACK( + "DBImpl::AtomicFlushMemTablesToOutputFiles:WaitCV", &res); + // + // If bg work is stopped, recovery thread first calls + // WaitForBackgroundWork() before proceeding to flush for recovery. This + // flush can block WaitForBackgroundWork() while waiting for recovery + // flush to install result. To avoid this deadlock, we should abort here + // if there is background error. + if (!flush_for_recovery && error_handler_.IsBGWorkStopped() && + !error_handler_.GetBGError().ok()) { + s = error_handler_.GetBGError(); + assert(!s.ok()); break; } atomic_flush_install_cv_.Wait(); - resuming_from_bg_err = - error_handler_.IsDBStopped() || - (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery || - bg_flush_args[0].flush_reason_ == - FlushReason::kErrorRecoveryRetryFlush); + resuming_from_bg_err = error_handler_.IsDBStopped() || flush_for_recovery; } if (!resuming_from_bg_err) { @@ -679,6 +768,17 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( // installation. s = error_handler_.GetRecoveryError(); } + // Since we are not installing these memtables, need to rollback + // to allow future flush job to pick up these memtables. + if (!s.ok()) { + for (int i = 0; i != num_cfs; ++i) { + assert(exec_status[i].first); + assert(exec_status[i].second.ok()); + auto& mems = jobs[i]->GetMemTables(); + cfds[i]->imm()->RollbackMemtableFlush( + mems, /*rollback_succeeding_memtables=*/false); + } + } } if (s.ok()) { @@ -782,7 +882,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles( // Need to undo atomic flush if something went wrong, i.e. s is not OK and // it is not because of CF drop. - if (!s.ok() && !s.IsColumnFamilyDropped()) { + if (!s.ok() && !s.IsColumnFamilyDropped() && !skip_set_bg_error) { if (log_io_s.ok()) { // Error while writing to MANIFEST. // In fact, versions_->io_status() can also be the result of renaming @@ -912,26 +1012,14 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options, end_without_ts, "" /*trim_ts*/); } - std::string begin_str; - std::string end_str; + std::string begin_str, end_str; + auto [begin, end] = + MaybeAddTimestampsToRange(begin_without_ts, end_without_ts, ts_sz, + &begin_str, &end_str, false /*exclusive_end*/); - // CompactRange compact all keys: [begin, end] inclusively. Add maximum - // timestamp to include all `begin` keys, and add minimal timestamp to include - // all `end` keys. - if (begin_without_ts != nullptr) { - AppendKeyWithMaxTimestamp(&begin_str, *begin_without_ts, ts_sz); - } - if (end_without_ts != nullptr) { - AppendKeyWithMinTimestamp(&end_str, *end_without_ts, ts_sz); - } - Slice begin(begin_str); - Slice end(end_str); - - Slice* begin_with_ts = begin_without_ts ? &begin : nullptr; - Slice* end_with_ts = end_without_ts ? &end : nullptr; - - return CompactRangeInternal(options, column_family, begin_with_ts, - end_with_ts, "" /*trim_ts*/); + return CompactRangeInternal( + options, column_family, begin.has_value() ? &begin.value() : nullptr, + end.has_value() ? &end.value() : nullptr, "" /*trim_ts*/); } Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, @@ -1438,7 +1526,8 @@ Status DBImpl::CompactFilesImpl( // without releasing the lock, so we're guaranteed a compaction can be formed. assert(c != nullptr); - c->SetInputVersion(version); + c->FinalizeInputInfo(version); + // deletion compaction currently not allowed in CompactFiles. assert(!c->deletion_compaction()); @@ -1488,7 +1577,12 @@ Status DBImpl::CompactFilesImpl( TEST_SYNC_POINT("CompactFilesImpl:3"); mutex_.Lock(); - Status status = compaction_job.Install(*c->mutable_cf_options()); + bool compaction_released = false; + Status status = + compaction_job.Install(*c->mutable_cf_options(), &compaction_released); + if (!compaction_released) { + c->ReleaseCompactionFiles(s); + } if (status.ok()) { assert(compaction_job.io_status().ok()); InstallSuperVersionAndScheduleWork(c->column_family_data(), @@ -1499,7 +1593,6 @@ Status DBImpl::CompactFilesImpl( // not check compaction_job.io_status() explicitly if we're not calling // SetBGError compaction_job.io_status().PermitUncheckedError(); - c->ReleaseCompactionFiles(s); // Need to make sure SstFileManager does its bookkeeping auto sfm = static_cast( immutable_db_options_.sst_file_manager.get()); @@ -1511,7 +1604,7 @@ Status DBImpl::CompactFilesImpl( if (compaction_job_info != nullptr) { BuildCompactionJobInfo(cfd, c.get(), s, compaction_job_stats, - job_context->job_id, version, compaction_job_info); + job_context->job_id, compaction_job_info); } if (status.ok()) { @@ -1608,21 +1701,18 @@ void DBImpl::NotifyOnCompactionBegin(ColumnFamilyData* cfd, Compaction* c, } c->SetNotifyOnCompactionCompleted(); - Version* current = cfd->current(); - current->Ref(); // release lock while notifying events mutex_.Unlock(); TEST_SYNC_POINT("DBImpl::NotifyOnCompactionBegin::UnlockMutex"); { CompactionJobInfo info{}; - BuildCompactionJobInfo(cfd, c, st, job_stats, job_id, current, &info); + BuildCompactionJobInfo(cfd, c, st, job_stats, job_id, &info); for (auto listener : immutable_db_options_.listeners) { listener->OnCompactionBegin(this, info); } info.status.PermitUncheckedError(); } mutex_.Lock(); - current->Unref(); } void DBImpl::NotifyOnCompactionCompleted( @@ -1640,21 +1730,17 @@ void DBImpl::NotifyOnCompactionCompleted( return; } - Version* current = cfd->current(); - current->Ref(); // release lock while notifying events mutex_.Unlock(); TEST_SYNC_POINT("DBImpl::NotifyOnCompactionCompleted::UnlockMutex"); { CompactionJobInfo info{}; - BuildCompactionJobInfo(cfd, c, st, compaction_job_stats, job_id, current, - &info); + BuildCompactionJobInfo(cfd, c, st, compaction_job_stats, job_id, &info); for (auto listener : immutable_db_options_.listeners) { listener->OnCompactionCompleted(this, info); } } mutex_.Lock(); - current->Unref(); // no need to signal bg_cv_ as it will be signaled at the end of the // flush process. } @@ -2150,7 +2236,8 @@ void DBImpl::GenerateFlushRequest(const autovector& cfds, // cfd may be null, see DBImpl::ScheduleFlushes continue; } - uint64_t max_memtable_id = cfd->imm()->GetLatestMemTableID(); + uint64_t max_memtable_id = cfd->imm()->GetLatestMemTableID( + immutable_db_options_.atomic_flush /* for_atomic_flush */); req->cfd_to_max_mem_id_to_persist.emplace(cfd, max_memtable_id); } } @@ -2194,15 +2281,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd, } WaitForPendingWrites(); - if (flush_reason != FlushReason::kErrorRecoveryRetryFlush && - (!cfd->mem()->IsEmpty() || !cached_recoverable_state_empty_.load())) { - // Note that, when flush reason is kErrorRecoveryRetryFlush, during the - // auto retry resume, we want to avoid creating new small memtables. - // Therefore, SwitchMemtable will not be called. Also, since ResumeImpl - // will iterate through all the CFs and call FlushMemtable during auto - // retry resume, it is possible that in some CFs, - // cfd->imm()->NumNotFlushed() = 0. In this case, so no flush request will - // be created and scheduled, status::OK() will be returned. + if (!cfd->mem()->IsEmpty() || !cached_recoverable_state_empty_.load()) { s = SwitchMemtable(cfd, &context); } const uint64_t flush_memtable_id = std::numeric_limits::max(); @@ -2211,10 +2290,10 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd, !cached_recoverable_state_empty_.load()) { FlushRequest req{flush_reason, {{cfd, flush_memtable_id}}}; flush_reqs.emplace_back(std::move(req)); - memtable_ids_to_wait.emplace_back(cfd->imm()->GetLatestMemTableID()); + memtable_ids_to_wait.emplace_back( + cfd->imm()->GetLatestMemTableID(false /* for_atomic_flush */)); } - if (immutable_db_options_.persist_stats_to_disk && - flush_reason != FlushReason::kErrorRecoveryRetryFlush) { + if (immutable_db_options_.persist_stats_to_disk) { ColumnFamilyData* cfd_stats = versions_->GetColumnFamilySet()->GetColumnFamily( kPersistentStatsColumnFamilyName); @@ -2240,7 +2319,8 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd, FlushRequest req{flush_reason, {{cfd_stats, flush_memtable_id}}}; flush_reqs.emplace_back(std::move(req)); memtable_ids_to_wait.emplace_back( - cfd_stats->imm()->GetLatestMemTableID()); + cfd_stats->imm()->GetLatestMemTableID( + false /* for_atomic_flush */)); } } } @@ -2291,8 +2371,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd, } s = WaitForFlushMemTables( cfds, flush_memtable_ids, - (flush_reason == FlushReason::kErrorRecovery || - flush_reason == FlushReason::kErrorRecoveryRetryFlush)); + flush_reason == FlushReason::kErrorRecovery /* resuming_from_bg_err */); InstrumentedMutexLock lock_guard(&mutex_); for (auto* tmp_cfd : cfds) { tmp_cfd->UnrefAndTryDelete(); @@ -2387,8 +2466,7 @@ Status DBImpl::AtomicFlushMemTables( } for (auto cfd : cfds) { - if ((cfd->mem()->IsEmpty() && cached_recoverable_state_empty_.load()) || - flush_reason == FlushReason::kErrorRecoveryRetryFlush) { + if (cfd->mem()->IsEmpty() && cached_recoverable_state_empty_.load()) { continue; } cfd->Ref(); @@ -2433,8 +2511,7 @@ Status DBImpl::AtomicFlushMemTables( } s = WaitForFlushMemTables( cfds, flush_memtable_ids, - (flush_reason == FlushReason::kErrorRecovery || - flush_reason == FlushReason::kErrorRecoveryRetryFlush)); + flush_reason == FlushReason::kErrorRecovery /* resuming_from_bg_err */); InstrumentedMutexLock lock_guard(&mutex_); for (auto* cfd : cfds) { cfd->UnrefAndTryDelete(); @@ -2443,6 +2520,68 @@ Status DBImpl::AtomicFlushMemTables( return s; } +Status DBImpl::RetryFlushesForErrorRecovery(FlushReason flush_reason, + bool wait) { + mutex_.AssertHeld(); + assert(flush_reason == FlushReason::kErrorRecoveryRetryFlush || + flush_reason == FlushReason::kCatchUpAfterErrorRecovery); + + // Collect referenced CFDs. + autovector cfds; + for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) { + if (!cfd->IsDropped() && cfd->initialized() && + cfd->imm()->NumNotFlushed() != 0) { + cfd->Ref(); + cfd->imm()->FlushRequested(); + cfds.push_back(cfd); + } + } + + // Submit flush requests for all immutable memtables needing flush. + // `flush_memtable_ids` will be populated such that all immutable + // memtables eligible for flush are waited on before this function + // returns. + autovector flush_memtable_ids; + if (immutable_db_options_.atomic_flush) { + FlushRequest flush_req; + GenerateFlushRequest(cfds, flush_reason, &flush_req); + SchedulePendingFlush(flush_req); + for (auto& iter : flush_req.cfd_to_max_mem_id_to_persist) { + flush_memtable_ids.push_back(iter.second); + } + } else { + for (auto cfd : cfds) { + flush_memtable_ids.push_back( + cfd->imm()->GetLatestMemTableID(false /* for_atomic_flush */)); + // Impose no bound on the highest memtable ID flushed. There is no + // reason to do so outside of atomic flush. + FlushRequest flush_req{ + flush_reason, + {{cfd, + std::numeric_limits::max() /* max_mem_id_to_persist */}}}; + SchedulePendingFlush(flush_req); + } + } + MaybeScheduleFlushOrCompaction(); + + Status s; + if (wait) { + mutex_.Unlock(); + autovector flush_memtable_id_ptrs; + for (auto& flush_memtable_id : flush_memtable_ids) { + flush_memtable_id_ptrs.push_back(&flush_memtable_id); + } + s = WaitForFlushMemTables(cfds, flush_memtable_id_ptrs, + true /* resuming_from_bg_err */); + mutex_.Lock(); + } + + for (auto* cfd : cfds) { + cfd->UnrefAndTryDelete(); + } + return s; +} + // Calling FlushMemTable(), whether from DB::Flush() or from Backup Engine, can // cause write stall, for example if one memtable is being flushed already. // This method tries to avoid write stall (similar to CompactRange() behavior) @@ -2506,8 +2645,11 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd, // check whether one extra immutable memtable or an extra L0 file would // cause write stalling mode to be entered. It could still enter stall // mode due to pending compaction bytes, but that's less common + // No extra immutable Memtable will be created if the current Memtable is + // empty. + int mem_to_flush = cfd->mem()->IsEmpty() ? 0 : 1; write_stall_condition = ColumnFamilyData::GetWriteStallConditionAndCause( - cfd->imm()->NumNotFlushed() + 1, + cfd->imm()->NumNotFlushed() + mem_to_flush, vstorage->l0_delay_trigger_count() + 1, vstorage->estimated_compaction_needed_bytes(), mutable_cf_options, *cfd->ioptions()) @@ -2641,6 +2783,7 @@ void DBImpl::EnableManualCompaction() { void DBImpl::MaybeScheduleFlushOrCompaction() { mutex_.AssertHeld(); + TEST_SYNC_POINT("DBImpl::MaybeScheduleFlushOrCompaction:Start"); if (!opened_successfully_) { // Compaction may introduce data race to DB open return; @@ -2653,6 +2796,11 @@ void DBImpl::MaybeScheduleFlushOrCompaction() { // There has been a hard error and this call is not part of the recovery // sequence. Bail out here so we don't get into an endless loop of // scheduling BG work which will again call this function + // + // Note that a non-recovery flush can still be scheduled if + // error_handler_.IsRecoveryInProgress() returns true. We rely on + // BackgroundCallFlush() to check flush reason and drop non-recovery + // flushes. return; } else if (shutting_down_.load(std::memory_order_acquire)) { // DB is being deleted; no more background compactions @@ -2819,6 +2967,9 @@ ColumnFamilyData* DBImpl::PickCompactionFromQueue( void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) { mutex_.AssertHeld(); + if (reject_new_background_jobs_) { + return; + } if (flush_req.cfd_to_max_mem_id_to_persist.empty()) { return; } @@ -2848,6 +2999,9 @@ void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) { void DBImpl::SchedulePendingCompaction(ColumnFamilyData* cfd) { mutex_.AssertHeld(); + if (reject_new_background_jobs_) { + return; + } if (!cfd->queued_for_compaction() && cfd->NeedsCompaction()) { AddToCompactionQueue(cfd); ++unscheduled_compactions_; @@ -2857,6 +3011,9 @@ void DBImpl::SchedulePendingCompaction(ColumnFamilyData* cfd) { void DBImpl::SchedulePendingPurge(std::string fname, std::string dir_to_sync, FileType type, uint64_t number, int job_id) { mutex_.AssertHeld(); + if (reject_new_background_jobs_) { + return; + } PurgeFileInfo file_info(fname, dir_to_sync, type, number, job_id); purge_files_.insert({{number, std::move(file_info)}}); } @@ -2945,6 +3102,7 @@ void DBImpl::UnscheduleFlushCallback(void* arg) { Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context, LogBuffer* log_buffer, FlushReason* reason, + bool* flush_rescheduled_to_retain_udt, Env::Priority thread_pri) { mutex_.AssertHeld(); @@ -2970,12 +3128,61 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context, autovector column_families_not_to_flush; while (!flush_queue_.empty()) { // This cfd is already referenced - auto [flush_reason, cfd_to_max_mem_id_to_persist] = - PopFirstFromFlushQueue(); + FlushRequest flush_req = PopFirstFromFlushQueue(); + FlushReason flush_reason = flush_req.flush_reason; + if (!error_handler_.GetBGError().ok() && error_handler_.IsBGWorkStopped() && + flush_reason != FlushReason::kErrorRecovery && + flush_reason != FlushReason::kErrorRecoveryRetryFlush) { + // Stop non-recovery flush when bg work is stopped + // Note that we drop the flush request here. + // Recovery thread should schedule further flushes after bg error + // is cleared. + status = error_handler_.GetBGError(); + assert(!status.ok()); + ROCKS_LOG_BUFFER(log_buffer, + "[JOB %d] Abort flush due to background error %s", + job_context->job_id, status.ToString().c_str()); + *reason = flush_reason; + for (auto item : flush_req.cfd_to_max_mem_id_to_persist) { + item.first->UnrefAndTryDelete(); + } + return status; + } + if (!immutable_db_options_.atomic_flush && + ShouldRescheduleFlushRequestToRetainUDT(flush_req)) { + assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1); + ColumnFamilyData* cfd = + flush_req.cfd_to_max_mem_id_to_persist.begin()->first; + if (cfd->UnrefAndTryDelete()) { + return Status::OK(); + } + ROCKS_LOG_BUFFER(log_buffer, + "FlushRequest for column family %s is re-scheduled to " + "retain user-defined timestamps.", + cfd->GetName().c_str()); + // Reschedule the `FlushRequest` as is without checking dropped column + // family etc. The follow-up job will do the check anyways, so save the + // duplication. Column family is deduplicated by `SchdulePendingFlush` and + // `PopFirstFromFlushQueue` contains at flush request enqueueing and + // dequeueing time. + // This flush request is rescheduled right after it's popped from the + // queue while the db mutex is held, so there should be no other + // FlushRequest for the same column family with higher `max_memtable_id` + // in the queue to block the reschedule from succeeding. +#ifndef NDEBUG + flush_req.reschedule_count += 1; +#endif /* !NDEBUG */ + SchedulePendingFlush(flush_req); + *reason = flush_reason; + *flush_rescheduled_to_retain_udt = true; + return Status::TryAgain(); + } superversion_contexts.clear(); - superversion_contexts.reserve(cfd_to_max_mem_id_to_persist.size()); + superversion_contexts.reserve( + flush_req.cfd_to_max_mem_id_to_persist.size()); - for (const auto& [cfd, max_memtable_id] : cfd_to_max_mem_id_to_persist) { + for (const auto& [cfd, max_memtable_id] : + flush_req.cfd_to_max_mem_id_to_persist) { if (cfd->GetMempurgeUsed()) { // If imm() contains silent memtables (e.g.: because // MemPurge was activated), requesting a flush will @@ -2992,7 +3199,13 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context, bg_flush_args.emplace_back(cfd, max_memtable_id, &(superversion_contexts.back()), flush_reason); } - if (!bg_flush_args.empty()) { + // `MaybeScheduleFlushOrCompaction` schedules as many `BackgroundCallFlush` + // jobs as the number of `FlushRequest` in the `flush_queue_`, a.k.a + // `unscheduled_flushes_`. So it's sufficient to make each `BackgroundFlush` + // handle one `FlushRequest` and each have a Status returned. + if (!bg_flush_args.empty() || !column_families_not_to_flush.empty()) { + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundFlush:CheckFlushRequest:cb", + const_cast(&flush_req.reschedule_count)); break; } } @@ -3054,11 +3267,20 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) { pending_outputs_inserted_elem(new std::list::iterator( CaptureCurrentFileNumberInPendingOutputs())); FlushReason reason; - - Status s = BackgroundFlush(&made_progress, &job_context, &log_buffer, - &reason, thread_pri); - if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped() && - reason != FlushReason::kErrorRecovery) { + bool flush_rescheduled_to_retain_udt = false; + Status s = + BackgroundFlush(&made_progress, &job_context, &log_buffer, &reason, + &flush_rescheduled_to_retain_udt, thread_pri); + if (s.IsTryAgain() && flush_rescheduled_to_retain_udt) { + bg_cv_.SignalAll(); // In case a waiter can proceed despite the error + mutex_.Unlock(); + TEST_SYNC_POINT_CALLBACK("DBImpl::AfterRetainUDTReschedule:cb", nullptr); + immutable_db_options_.clock->SleepForMicroseconds( + 100000); // prevent hot loop + mutex_.Lock(); + } else if (!s.ok() && !s.IsShutdownInProgress() && + !s.IsColumnFamilyDropped() && + reason != FlushReason::kErrorRecovery) { // Wait a little bit before retrying background flush in // case this is an environmental problem and we do not want to // chew up resources for failed flushes for the duration of @@ -3068,9 +3290,9 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) { bg_cv_.SignalAll(); // In case a waiter can proceed despite the error mutex_.Unlock(); ROCKS_LOG_ERROR(immutable_db_options_.info_log, - "Waiting after background flush error: %s" + "[JOB %d] Waiting after background flush error: %s" "Accumulated background error counts: %" PRIu64, - s.ToString().c_str(), error_cnt); + job_context.job_id, s.ToString().c_str(), error_cnt); log_buffer.FlushBufferToLog(); LogFlush(immutable_db_options_.info_log); immutable_db_options_.clock->SleepForMicroseconds(1000000); @@ -3079,11 +3301,15 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) { TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FlushFinish:0"); ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem); - - // If flush failed, we want to delete all temporary files that we might have - // created. Thus, we force full scan in FindObsoleteFiles() - FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() && - !s.IsColumnFamilyDropped()); + // There is no need to find obsolete files if the flush job is rescheduled + // to retain user-defined timestamps because the job doesn't get to the + // stage of actually flushing the MemTables. + if (!flush_rescheduled_to_retain_udt) { + // If flush failed, we want to delete all temporary files that we might + // have created. Thus, we force full scan in FindObsoleteFiles() + FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() && + !s.IsColumnFamilyDropped()); + } // delete unnecessary files if any, this is done outside the mutex if (job_context.HaveSomethingToClean() || job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) { @@ -3308,8 +3534,6 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, std::unique_ptr task_token; - // InternalKey manual_end_storage; - // InternalKey* manual_end = &manual_end_storage; bool sfm_reserved_compact_space = false; if (is_manual) { ManualCompactionState* m = manual_compaction; @@ -3445,6 +3669,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, } IOStatus io_s; + bool compaction_released = false; if (!c) { // Nothing to do ROCKS_LOG_BUFFER(log_buffer, "Compaction nothing to do"); @@ -3467,7 +3692,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, } status = versions_->LogAndApply( c->column_family_data(), *c->mutable_cf_options(), read_options, - c->edit(), &mutex_, directories_.GetDbDir()); + c->edit(), &mutex_, directories_.GetDbDir(), + /*new_descriptor_log=*/false, /*column_family_options=*/nullptr, + [&c, &compaction_released](const Status& s) { + c->ReleaseCompactionFiles(s); + compaction_released = true; + }); io_s = versions_->io_status(); InstallSuperVersionAndScheduleWork(c->column_family_data(), &job_context->superversion_contexts[0], @@ -3475,6 +3705,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, ROCKS_LOG_BUFFER(log_buffer, "[%s] Deleted %d files\n", c->column_family_data()->GetName().c_str(), c->num_input_files(0)); + if (status.ok() && io_s.ok()) { + UpdateDeletionCompactionStats(c); + } *made_progress = true; TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:AfterCompaction", c->column_family_data()); @@ -3533,7 +3766,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, } status = versions_->LogAndApply( c->column_family_data(), *c->mutable_cf_options(), read_options, - c->edit(), &mutex_, directories_.GetDbDir()); + c->edit(), &mutex_, directories_.GetDbDir(), + /*new_descriptor_log=*/false, /*column_family_options=*/nullptr, + [&c, &compaction_released](const Status& s) { + c->ReleaseCompactionFiles(s); + compaction_released = true; + }); io_s = versions_->io_status(); // Use latest MutableCFOptions InstallSuperVersionAndScheduleWork(c->column_family_data(), @@ -3583,6 +3821,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, // Transfer requested token, so it doesn't need to do it again. ca->prepicked_compaction->task_token = std::move(task_token); ++bg_bottom_compaction_scheduled_; + assert(c == nullptr); env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM, this, &DBImpl::UnscheduleCompactionCallback); } else { @@ -3626,8 +3865,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, compaction_job.Run().PermitUncheckedError(); TEST_SYNC_POINT("DBImpl::BackgroundCompaction:NonTrivial:AfterRun"); mutex_.Lock(); - - status = compaction_job.Install(*c->mutable_cf_options()); + status = + compaction_job.Install(*c->mutable_cf_options(), &compaction_released); io_s = compaction_job.io_status(); if (status.ok()) { InstallSuperVersionAndScheduleWork(c->column_family_data(), @@ -3646,7 +3885,23 @@ Status DBImpl::BackgroundCompaction(bool* made_progress, } if (c != nullptr) { - c->ReleaseCompactionFiles(status); + if (!compaction_released) { + c->ReleaseCompactionFiles(status); + } else { +#ifndef NDEBUG + // Sanity checking that compaction files are freed. + for (size_t i = 0; i < c->num_input_levels(); i++) { + for (size_t j = 0; j < c->inputs(i)->size(); j++) { + assert(!c->input(i, j)->being_compacted); + } + } + std::unordered_set* cip = c->column_family_data() + ->compaction_picker() + ->compactions_in_progress(); + assert(cip->find(c.get()) == cip->end()); +#endif + } + *made_progress = true; // Need to make sure SstFileManager does its bookkeeping @@ -3831,10 +4086,31 @@ bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) { return false; } +void DBImpl::UpdateDeletionCompactionStats( + const std::unique_ptr& c) { + if (c == nullptr) { + return; + } + + CompactionReason reason = c->compaction_reason(); + + switch (reason) { + case CompactionReason::kFIFOMaxSize: + RecordTick(stats_, FIFO_MAX_SIZE_COMPACTIONS); + break; + case CompactionReason::kFIFOTtl: + RecordTick(stats_, FIFO_TTL_COMPACTIONS); + break; + default: + assert(false); + break; + } +} + void DBImpl::BuildCompactionJobInfo( const ColumnFamilyData* cfd, Compaction* c, const Status& st, const CompactionJobStats& compaction_job_stats, const int job_id, - const Version* current, CompactionJobInfo* compaction_job_info) const { + CompactionJobInfo* compaction_job_info) const { assert(compaction_job_info != nullptr); compaction_job_info->cf_id = cfd->GetID(); compaction_job_info->cf_name = cfd->GetName(); @@ -3844,7 +4120,12 @@ void DBImpl::BuildCompactionJobInfo( compaction_job_info->base_input_level = c->start_level(); compaction_job_info->output_level = c->output_level(); compaction_job_info->stats = compaction_job_stats; - compaction_job_info->table_properties = c->GetOutputTableProperties(); + const auto& input_table_properties = c->GetInputTableProperties(); + const auto& output_table_properties = c->GetOutputTableProperties(); + compaction_job_info->table_properties.insert(input_table_properties.begin(), + input_table_properties.end()); + compaction_job_info->table_properties.insert(output_table_properties.begin(), + output_table_properties.end()); compaction_job_info->compaction_reason = c->compaction_reason(); compaction_job_info->compression = c->output_compression(); @@ -3858,15 +4139,9 @@ void DBImpl::BuildCompactionJobInfo( compaction_job_info->input_files.push_back(fn); compaction_job_info->input_file_infos.push_back(CompactionFileInfo{ static_cast(i), file_number, fmd->oldest_blob_file_number}); - if (compaction_job_info->table_properties.count(fn) == 0) { - std::shared_ptr tp; - auto s = current->GetTableProperties(read_options, &tp, fmd, &fn); - if (s.ok()) { - compaction_job_info->table_properties[fn] = tp; - } - } } } + for (const auto& newf : c->edit()->GetNewFiles()) { const FileMetaData& meta = newf.second; const FileDescriptor& desc = meta.fd; @@ -4019,8 +4294,18 @@ Status DBImpl::WaitForCompact( if (!s.ok()) { return s; } + } else if (wait_for_compact_options.close_db && + has_unpersisted_data_.load(std::memory_order_relaxed) && + !mutable_db_options_.avoid_flush_during_shutdown) { + Status s = + DBImpl::FlushAllColumnFamilies(FlushOptions(), FlushReason::kShutDown); + if (!s.ok()) { + return s; + } } TEST_SYNC_POINT("DBImpl::WaitForCompact:StartWaiting"); + const auto deadline = immutable_db_options_.clock->NowMicros() + + wait_for_compact_options.timeout.count(); for (;;) { if (shutting_down_.load(std::memory_order_acquire)) { return Status::ShutdownInProgress(); @@ -4030,9 +4315,24 @@ Status DBImpl::WaitForCompact( } if ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ || bg_flush_scheduled_ || unscheduled_compactions_ || - unscheduled_flushes_) && + unscheduled_flushes_ || error_handler_.IsRecoveryInProgress()) && (error_handler_.GetBGError().ok())) { - bg_cv_.Wait(); + if (wait_for_compact_options.timeout.count()) { + if (bg_cv_.TimedWait(deadline)) { + return Status::TimedOut(); + } + } else { + bg_cv_.Wait(); + } + } else if (wait_for_compact_options.close_db) { + reject_new_background_jobs_ = true; + mutex_.Unlock(); + Status s = Close(); + mutex_.Lock(); + if (!s.ok()) { + reject_new_background_jobs_ = false; + } + return s; } else { return error_handler_.GetBGError(); } diff --git a/db/db_impl/db_impl_debug.cc b/db/db_impl/db_impl_debug.cc index be63637a2..17050e465 100644 --- a/db/db_impl/db_impl_debug.cc +++ b/db/db_impl/db_impl_debug.cc @@ -306,11 +306,16 @@ const PeriodicTaskScheduler& DBImpl::TEST_GetPeriodicTaskScheduler() const { SeqnoToTimeMapping DBImpl::TEST_GetSeqnoToTimeMapping() const { InstrumentedMutexLock l(&mutex_); - return seqno_time_mapping_; + return seqno_to_time_mapping_; } +const autovector& DBImpl::TEST_GetFilesToQuarantine() const { + InstrumentedMutexLock l(&mutex_); + return error_handler_.GetFilesToQuarantine(); +} size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const { + InstrumentedMutexLock l(&const_cast(this)->stats_history_mutex_); return EstimateInMemoryStatsHistorySize(); } } // namespace ROCKSDB_NAMESPACE diff --git a/db/db_impl/db_impl_files.cc b/db/db_impl/db_impl_files.cc index 9fc9bb27a..bd4879647 100644 --- a/db/db_impl/db_impl_files.cc +++ b/db/db_impl/db_impl_files.cc @@ -146,6 +146,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force, // mutex_ cannot be released. Otherwise, we might see no min_pending_output // here but later find newer generated unfinalized files while scanning. job_context->min_pending_output = MinObsoleteSstNumberToKeep(); + job_context->files_to_quarantine = error_handler_.GetFilesToQuarantine(); // Get obsolete files. This function will also update the list of // pending files in VersionSet(). @@ -419,6 +420,8 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) { state.blob_live.end()); std::unordered_set log_recycle_files_set( state.log_recycle_files.begin(), state.log_recycle_files.end()); + std::unordered_set quarantine_files_set( + state.files_to_quarantine.begin(), state.files_to_quarantine.end()); auto candidate_files = state.full_scan_candidate_files; candidate_files.reserve( @@ -457,12 +460,12 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) { std::sort(candidate_files.begin(), candidate_files.end(), [](const JobContext::CandidateFileInfo& lhs, const JobContext::CandidateFileInfo& rhs) { - if (lhs.file_name > rhs.file_name) { + if (lhs.file_name < rhs.file_name) { return true; - } else if (lhs.file_name < rhs.file_name) { + } else if (lhs.file_name > rhs.file_name) { return false; } else { - return (lhs.file_path > rhs.file_path); + return (lhs.file_path < rhs.file_path); } }); candidate_files.erase( @@ -522,6 +525,10 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) { continue; } + if (quarantine_files_set.find(number) != quarantine_files_set.end()) { + continue; + } + bool keep = true; switch (type) { case kWalFile: @@ -995,7 +1002,7 @@ Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) { if (type == kTableFile && number >= next_file_number && recovery_ctx->files_to_delete_.find(normalized_fpath) == recovery_ctx->files_to_delete_.end()) { - recovery_ctx->files_to_delete_.emplace(normalized_fpath); + recovery_ctx->files_to_delete_.emplace(normalized_fpath, path); } } } diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 988283381..074fa8621 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -25,6 +25,7 @@ #include "rocksdb/wal_filter.h" #include "test_util/sync_point.h" #include "util/rate_limiter_impl.h" +#include "util/string_util.h" #include "util/udt_util.h" namespace ROCKSDB_NAMESPACE { @@ -143,11 +144,6 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src, result.wal_dir = result.wal_dir.substr(0, result.wal_dir.size() - 1); } - if (result.use_direct_reads && result.compaction_readahead_size == 0) { - TEST_SYNC_POINT_CALLBACK("SanitizeOptions:direct_io", nullptr); - result.compaction_readahead_size = 1024 * 1024 * 2; - } - // Force flush on DB open if 2PC is enabled, since with 2PC we have no // guarantee that consecutive log files have consecutive sequence id, which // make recovery complicated. @@ -296,6 +292,18 @@ Status DBImpl::ValidateOptions(const DBOptions& db_options) { "writes in direct IO require writable_file_max_buffer_size > 0"); } + if (db_options.daily_offpeak_time_utc != "") { + int start_time, end_time; + if (!TryParseTimeRangeString(db_options.daily_offpeak_time_utc, start_time, + end_time)) { + return Status::InvalidArgument( + "daily_offpeak_time_utc should be set in the format HH:mm-HH:mm " + "(e.g. 04:30-07:30)"); + } else if (start_time == end_time) { + return Status::InvalidArgument( + "start_time and end_time cannot be the same"); + } + } return Status::OK(); } @@ -410,7 +418,8 @@ Status DBImpl::Recover( uint64_t* recovered_seq, RecoveryContext* recovery_ctx) { mutex_.AssertHeld(); - bool is_new_db = false; + bool tmp_is_new_db = false; + bool& is_new_db = recovery_ctx ? recovery_ctx->is_new_db_ : tmp_is_new_db; assert(db_lock_ == nullptr); std::vector files_in_dbname; if (!read_only) { @@ -863,7 +872,8 @@ Status DBImpl::PersistentStatsProcessFormatVersion() { if (s.ok()) { ColumnFamilyOptions cfo; OptimizeForPersistentStats(&cfo); - s = CreateColumnFamily(cfo, kPersistentStatsColumnFamilyName, &handle); + s = CreateColumnFamilyImpl(cfo, kPersistentStatsColumnFamilyName, + &handle); } if (s.ok()) { persist_stats_cf_handle_ = static_cast(handle); @@ -916,7 +926,7 @@ Status DBImpl::InitPersistStatsColumnFamily() { ColumnFamilyHandle* handle = nullptr; ColumnFamilyOptions cfo; OptimizeForPersistentStats(&cfo); - s = CreateColumnFamily(cfo, kPersistentStatsColumnFamilyName, &handle); + s = CreateColumnFamilyImpl(cfo, kPersistentStatsColumnFamilyName, &handle); persist_stats_cf_handle_ = static_cast(handle); mutex_.Lock(); } @@ -932,8 +942,11 @@ Status DBImpl::LogAndApplyForRecovery(const RecoveryContext& recovery_ctx) { recovery_ctx.edit_lists_, &mutex_, directories_.GetDbDir()); if (s.ok() && !(recovery_ctx.files_to_delete_.empty())) { mutex_.Unlock(); - for (const auto& fname : recovery_ctx.files_to_delete_) { - s = env_->DeleteFile(fname); + for (const auto& stale_sst_file : recovery_ctx.files_to_delete_) { + s = DeleteDBFile(&immutable_db_options_, stale_sst_file.first, + stale_sst_file.second, + /*force_bg=*/false, + /*force_fg=*/false); if (!s.ok()) { break; } @@ -1203,10 +1216,10 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, Status::Corruption("log record too small")); continue; } - // We create a new batch and initialize with a valid prot_info_ to store // the data checksums WriteBatch batch; + std::unique_ptr new_batch; status = WriteBatchInternal::SetContents(&batch, record); if (!status.ok()) { @@ -1215,26 +1228,36 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, const UnorderedMap& record_ts_sz = reader.GetRecordedTimestampSize(); - // TODO(yuzhangyu): update mode to kReconcileInconsistency when user - // comparator can be changed. status = HandleWriteBatchTimestampSizeDifference( &batch, running_ts_sz, record_ts_sz, - TimestampSizeConsistencyMode::kVerifyConsistency); + TimestampSizeConsistencyMode::kReconcileInconsistency, &new_batch); if (!status.ok()) { return status; } + + bool batch_updated = new_batch != nullptr; + WriteBatch* batch_to_use = batch_updated ? new_batch.get() : &batch; TEST_SYNC_POINT_CALLBACK( - "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:batch", &batch); + "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:batch", + batch_to_use); TEST_SYNC_POINT_CALLBACK( "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:checksum", &record_checksum); status = WriteBatchInternal::UpdateProtectionInfo( - &batch, 8 /* bytes_per_key */, &record_checksum); + batch_to_use, 8 /* bytes_per_key */, + batch_updated ? nullptr : &record_checksum); if (!status.ok()) { return status; } - SequenceNumber sequence = WriteBatchInternal::Sequence(&batch); + SequenceNumber sequence = WriteBatchInternal::Sequence(batch_to_use); + if (sequence > kMaxSequenceNumber) { + reporter.Corruption( + record.size(), + Status::Corruption("sequence " + std::to_string(sequence) + + " is too large")); + continue; + } if (immutable_db_options_.wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery) { @@ -1255,7 +1278,7 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, // and returns true. if (!InvokeWalFilterIfNeededOnWalRecord(wal_number, fname, reporter, status, stop_replay_by_wal_filter, - batch)) { + *batch_to_use)) { continue; } @@ -1266,7 +1289,7 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, // That's why we set ignore missing column families to true bool has_valid_writes = false; status = WriteBatchInternal::InsertInto( - &batch, column_family_memtables_.get(), &flush_scheduler_, + batch_to_use, column_family_memtables_.get(), &flush_scheduler_, &trim_history_scheduler_, true, wal_number, this, false /* concurrent_memtable_writes */, next_sequence, &has_valid_writes, seq_per_batch_, batch_per_txn_); @@ -1300,7 +1323,7 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, flushed = true; cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(), - *next_sequence); + *next_sequence - 1); } } } @@ -1651,7 +1674,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, TableFileCreationReason::kRecovery, 0 /* oldest_key_time */, 0 /* file_creation_time */, db_id_, db_session_id_, 0 /* target_file_size */, meta.fd.GetNumber()); - SeqnoToTimeMapping empty_seqno_time_mapping; + SeqnoToTimeMapping empty_seqno_to_time_mapping; Version* version = cfd->current(); version->Ref(); const ReadOptions read_option(Env::IOActivity::kDBOpen); @@ -1663,7 +1686,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, snapshot_seqs, earliest_write_conflict_snapshot, kMaxSequenceNumber, snapshot_checker, paranoid_file_checks, cfd->internal_stats(), &io_s, io_tracer_, BlobFileCreationReason::kRecovery, - empty_seqno_time_mapping, &event_logger_, job_id, Env::IO_HIGH, + empty_seqno_to_time_mapping, &event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */, write_hint, nullptr /*full_history_ts_low*/, &blob_callback_, version, &num_input_entries); @@ -1717,6 +1740,22 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, for (const auto& blob : blob_file_additions) { edit->AddBlobFile(blob); } + + // For UDT in memtable only feature, move up the cutoff timestamp whenever + // a flush happens. + const Comparator* ucmp = cfd->user_comparator(); + size_t ts_sz = ucmp->timestamp_size(); + if (ts_sz > 0 && !cfd->ioptions()->persist_user_defined_timestamps) { + Slice mem_newest_udt = mem->GetNewestUDT(); + std::string full_history_ts_low = cfd->GetFullHistoryTsLow(); + if (full_history_ts_low.empty() || + ucmp->CompareTimestamp(mem_newest_udt, full_history_ts_low) >= 0) { + std::string new_full_history_ts_low; + GetFullHistoryTsLowFromU64CutoffTs(&mem_newest_udt, + &new_full_history_ts_low); + edit->SetFullHistoryTsLow(new_full_history_ts_low); + } + } } InternalStats::CompactionStats stats(CompactionReason::kFlush, 1); @@ -1961,6 +2000,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, impl->wal_in_db_path_ = impl->immutable_db_options_.IsWalDirSameAsDBPath(); RecoveryContext recovery_ctx; + impl->options_mutex_.Lock(); impl->mutex_.Lock(); // Handles create_if_missing, error_if_exists @@ -2042,7 +2082,9 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, // missing column family, create it ColumnFamilyHandle* handle = nullptr; impl->mutex_.Unlock(); - s = impl->CreateColumnFamily(cf.options, cf.name, &handle); + // NOTE: the work normally done in WrapUpCreateColumnFamilies will + // be done separately below. + s = impl->CreateColumnFamilyImpl(cf.options, cf.name, &handle); impl->mutex_.Lock(); if (s.ok()) { handles->push_back(handle); @@ -2093,9 +2135,8 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, if (s.ok()) { // Persist RocksDB Options before scheduling the compaction. // The WriteOptionsFile() will release and lock the mutex internally. - persist_options_status = impl->WriteOptionsFile( - false /*need_mutex_lock*/, false /*need_enter_write_thread*/); - + persist_options_status = + impl->WriteOptionsFile(true /*db_mutex_already_held*/); *dbptr = impl; impl->opened_successfully_ = true; impl->DeleteObsoleteFiles(); @@ -2216,10 +2257,10 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, if (s.ok()) { s = impl->StartPeriodicTaskScheduler(); } - if (s.ok()) { - s = impl->RegisterRecordSeqnoTimeWorker(); + s = impl->RegisterRecordSeqnoTimeWorker(recovery_ctx.is_new_db_); } + impl->options_mutex_.Unlock(); if (!s.ok()) { for (auto* h : *handles) { delete h; diff --git a/db/db_impl/db_impl_readonly.cc b/db/db_impl/db_impl_readonly.cc index 6f7e95fa7..997a4e2ed 100644 --- a/db/db_impl/db_impl_readonly.cc +++ b/db/db_impl/db_impl_readonly.cc @@ -29,36 +29,23 @@ DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options, DBImplReadOnly::~DBImplReadOnly() {} // Implementations of the DB interface -Status DBImplReadOnly::Get(const ReadOptions& read_options, - ColumnFamilyHandle* column_family, const Slice& key, - PinnableSlice* pinnable_val) { - return Get(read_options, column_family, key, pinnable_val, - /*timestamp*/ nullptr); -} +Status DBImplReadOnly::GetImpl(const ReadOptions& read_options, + const Slice& key, + GetImplOptions& get_impl_options) { + assert(get_impl_options.value != nullptr || + get_impl_options.columns != nullptr); + assert(get_impl_options.column_family); -Status DBImplReadOnly::Get(const ReadOptions& read_options, - ColumnFamilyHandle* column_family, const Slice& key, - PinnableSlice* pinnable_val, - std::string* timestamp) { - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - } - assert(pinnable_val != nullptr); - PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); - StopWatch sw(immutable_db_options_.clock, stats_, DB_GET); - PERF_TIMER_GUARD(get_snapshot_time); + Status s; - assert(column_family); if (read_options.timestamp) { - const Status s = FailIfTsMismatchCf( - column_family, *(read_options.timestamp), /*ts_for_read=*/true); + s = FailIfTsMismatchCf(get_impl_options.column_family, + *(read_options.timestamp)); if (!s.ok()) { return s; } } else { - const Status s = FailIfCfHasTs(column_family); + s = FailIfCfHasTs(get_impl_options.column_family); if (!s.ok()) { return s; } @@ -66,67 +53,99 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options, // Clear the timestamps for returning results so that we can distinguish // between tombstone or key that has never been written - if (timestamp) { - timestamp->clear(); + if (get_impl_options.timestamp) { + get_impl_options.timestamp->clear(); } - const Comparator* ucmp = column_family->GetComparator(); - assert(ucmp); - std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr; + PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); + StopWatch sw(immutable_db_options_.clock, stats_, DB_GET); + PERF_TIMER_GUARD(get_snapshot_time); - Status s; + const Comparator* ucmp = get_impl_options.column_family->GetComparator(); + assert(ucmp); + std::string* ts = + ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr; SequenceNumber snapshot = versions_->LastSequence(); GetWithTimestampReadCallback read_cb(snapshot); - auto cfh = static_cast_with_check(column_family); + auto cfh = static_cast_with_check( + get_impl_options.column_family); auto cfd = cfh->cfd(); if (tracer_) { InstrumentedMutexLock lock(&trace_mutex_); if (tracer_) { - tracer_->Get(column_family, key); + tracer_->Get(get_impl_options.column_family, key); } } + + // In read-only mode Get(), no super version operation is needed (i.e. + // GetAndRefSuperVersion and ReturnAndCleanupSuperVersion) SuperVersion* super_version = cfd->GetSuperVersion(); + if (read_options.timestamp && read_options.timestamp->size() > 0) { + s = FailIfReadCollapsedHistory(cfd, super_version, + *(read_options.timestamp)); + if (!s.ok()) { + return s; + } + } MergeContext merge_context; SequenceNumber max_covering_tombstone_seq = 0; LookupKey lkey(key, snapshot, read_options.timestamp); PERF_TIMER_STOP(get_snapshot_time); - if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), - /*columns=*/nullptr, ts, &s, &merge_context, - &max_covering_tombstone_seq, read_options, - false /* immutable_memtable */, &read_cb)) { - pinnable_val->PinSelf(); + + // Look up starts here + if (super_version->mem->Get( + lkey, + get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr, + get_impl_options.columns, ts, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, &read_cb)) { + if (get_impl_options.value) { + get_impl_options.value->PinSelf(); + } RecordTick(stats_, MEMTABLE_HIT); } else { PERF_TIMER_GUARD(get_from_output_files_time); PinnedIteratorsManager pinned_iters_mgr; super_version->current->Get( - read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s, - &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr, + read_options, lkey, get_impl_options.value, get_impl_options.columns, + ts, &s, &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr, /*value_found*/ nullptr, /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, /*is_blob*/ nullptr, /*do_merge*/ true); RecordTick(stats_, MEMTABLE_MISS); } - RecordTick(stats_, NUMBER_KEYS_READ); - size_t size = pinnable_val->size(); - RecordTick(stats_, BYTES_READ, size); - RecordInHistogram(stats_, BYTES_PER_READ, size); - PERF_COUNTER_ADD(get_read_bytes, size); + { + RecordTick(stats_, NUMBER_KEYS_READ); + size_t size = 0; + if (get_impl_options.value) { + size = get_impl_options.value->size(); + } else if (get_impl_options.columns) { + size = get_impl_options.columns->serialized_size(); + } + RecordTick(stats_, BYTES_READ, size); + RecordInHistogram(stats_, BYTES_PER_READ, size); + PERF_COUNTER_ADD(get_read_bytes, size); + } return s; } -Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options, +Iterator* DBImplReadOnly::NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) { - if (read_options.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { return NewErrorIterator(Status::InvalidArgument( - "Cannot call NewIterator with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`")); + "Can only call NewIterator with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`")); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; } assert(column_family); if (read_options.timestamp) { - const Status s = FailIfTsMismatchCf( - column_family, *(read_options.timestamp), /*ts_for_read=*/true); + const Status s = + FailIfTsMismatchCf(column_family, *(read_options.timestamp)); if (!s.ok()) { return NewErrorIterator(s); } @@ -139,6 +158,14 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options, auto cfh = static_cast_with_check(column_family); auto cfd = cfh->cfd(); SuperVersion* super_version = cfd->GetSuperVersion()->Ref(); + if (read_options.timestamp && read_options.timestamp->size() > 0) { + const Status s = FailIfReadCollapsedHistory(cfd, super_version, + *(read_options.timestamp)); + if (!s.ok()) { + cfd->GetSuperVersion()->Unref(); + return NewErrorIterator(s); + } + } SequenceNumber latest_snapshot = versions_->LastSequence(); SequenceNumber read_seq = read_options.snapshot != nullptr @@ -165,8 +192,7 @@ Status DBImplReadOnly::NewIterators( if (read_options.timestamp) { for (auto* cf : column_families) { assert(cf); - const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp), - /*ts_for_read=*/true); + const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp)); if (!s.ok()) { return s; } @@ -194,9 +220,27 @@ Status DBImplReadOnly::NewIterators( ->number_ : latest_snapshot; + autovector> cfd_to_sv; + + const bool check_read_ts = + read_options.timestamp && read_options.timestamp->size() > 0; for (auto cfh : column_families) { auto* cfd = static_cast_with_check(cfh)->cfd(); auto* sv = cfd->GetSuperVersion()->Ref(); + cfd_to_sv.emplace_back(cfd, sv); + if (check_read_ts) { + const Status s = + FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp)); + if (!s.ok()) { + for (auto prev_entry : cfd_to_sv) { + std::get<1>(prev_entry)->Unref(); + } + return s; + } + } + } + assert(cfd_to_sv.size() == column_families.size()); + for (auto [cfd, sv] : cfd_to_sv) { auto* db_iter = NewArenaWrappedDbIterator( env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, sv->current, read_seq, diff --git a/db/db_impl/db_impl_readonly.h b/db/db_impl/db_impl_readonly.h index ccd52e407..32bc85607 100644 --- a/db/db_impl/db_impl_readonly.h +++ b/db/db_impl/db_impl_readonly.h @@ -24,18 +24,14 @@ class DBImplReadOnly : public DBImpl { virtual ~DBImplReadOnly(); // Implementations of the DB interface - using DB::Get; - virtual Status Get(const ReadOptions& options, - ColumnFamilyHandle* column_family, const Slice& key, - PinnableSlice* value) override; - Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* value, - std::string* timestamp) override; + using DBImpl::GetImpl; + Status GetImpl(const ReadOptions& options, const Slice& key, + GetImplOptions& get_impl_options) override; // TODO: Implement ReadOnly MultiGet? using DBImpl::NewIterator; - virtual Iterator* NewIterator(const ReadOptions&, + virtual Iterator* NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) override; virtual Status NewIterators( @@ -57,6 +53,10 @@ class DBImplReadOnly : public DBImpl { const WideColumns& /* columns */) override { return Status::NotSupported("Not supported operation in read only mode."); } + Status PutEntity(const WriteOptions& /* options */, const Slice& /* key */, + const AttributeGroups& /* attribute_groups */) override { + return Status::NotSupported("Not supported operation in read only mode."); + } using DBImpl::Merge; virtual Status Merge(const WriteOptions& /*options*/, @@ -179,4 +179,3 @@ class DBImplReadOnly : public DBImpl { friend class DB; }; } // namespace ROCKSDB_NAMESPACE - diff --git a/db/db_impl/db_impl_secondary.cc b/db/db_impl/db_impl_secondary.cc index de3536841..235a528ba 100644 --- a/db/db_impl/db_impl_secondary.cc +++ b/db/db_impl/db_impl_secondary.cc @@ -339,90 +339,93 @@ Status DBImplSecondary::RecoverLogFiles( return status; } -// Implementation of the DB interface -Status DBImplSecondary::Get(const ReadOptions& read_options, - ColumnFamilyHandle* column_family, const Slice& key, - PinnableSlice* value) { - return GetImpl(read_options, column_family, key, value, - /*timestamp*/ nullptr); -} - -Status DBImplSecondary::Get(const ReadOptions& read_options, - ColumnFamilyHandle* column_family, const Slice& key, - PinnableSlice* value, std::string* timestamp) { - return GetImpl(read_options, column_family, key, value, timestamp); -} - Status DBImplSecondary::GetImpl(const ReadOptions& read_options, - ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* pinnable_val, - std::string* timestamp) { - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - } - assert(pinnable_val != nullptr); - PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); - StopWatch sw(immutable_db_options_.clock, stats_, DB_GET); - PERF_TIMER_GUARD(get_snapshot_time); + const Slice& key, + GetImplOptions& get_impl_options) { + assert(get_impl_options.value != nullptr || + get_impl_options.columns != nullptr); + assert(get_impl_options.column_family); + + Status s; - assert(column_family); if (read_options.timestamp) { - const Status s = FailIfTsMismatchCf( - column_family, *(read_options.timestamp), /*ts_for_read=*/true); + s = FailIfTsMismatchCf(get_impl_options.column_family, + *(read_options.timestamp)); if (!s.ok()) { return s; } } else { - const Status s = FailIfCfHasTs(column_family); + s = FailIfCfHasTs(get_impl_options.column_family); if (!s.ok()) { return s; } } - // Clear the timestamp for returning results so that we can distinguish - // between tombstone or key that has never been written later. - if (timestamp) { - timestamp->clear(); + // Clear the timestamps for returning results so that we can distinguish + // between tombstone or key that has never been written + if (get_impl_options.timestamp) { + get_impl_options.timestamp->clear(); } - auto cfh = static_cast(column_family); - ColumnFamilyData* cfd = cfh->cfd(); + PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); + StopWatch sw(immutable_db_options_.clock, stats_, DB_GET); + PERF_TIMER_GUARD(get_snapshot_time); + + const Comparator* ucmp = get_impl_options.column_family->GetComparator(); + assert(ucmp); + std::string* ts = + ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr; + SequenceNumber snapshot = versions_->LastSequence(); + GetWithTimestampReadCallback read_cb(snapshot); + auto cfh = static_cast_with_check( + get_impl_options.column_family); + auto cfd = cfh->cfd(); if (tracer_) { InstrumentedMutexLock lock(&trace_mutex_); if (tracer_) { - tracer_->Get(column_family, key); + tracer_->Get(get_impl_options.column_family, key); } } + // Acquire SuperVersion SuperVersion* super_version = GetAndRefSuperVersion(cfd); - SequenceNumber snapshot = versions_->LastSequence(); - GetWithTimestampReadCallback read_cb(snapshot); + if (read_options.timestamp && read_options.timestamp->size() > 0) { + s = FailIfReadCollapsedHistory(cfd, super_version, + *(read_options.timestamp)); + if (!s.ok()) { + ReturnAndCleanupSuperVersion(cfd, super_version); + return s; + } + } MergeContext merge_context; SequenceNumber max_covering_tombstone_seq = 0; - Status s; LookupKey lkey(key, snapshot, read_options.timestamp); PERF_TIMER_STOP(get_snapshot_time); - bool done = false; - const Comparator* ucmp = column_family->GetComparator(); - assert(ucmp); - std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr; - if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), - /*columns=*/nullptr, ts, &s, &merge_context, - &max_covering_tombstone_seq, read_options, - false /* immutable_memtable */, &read_cb)) { + + // Look up starts here + if (super_version->mem->Get( + lkey, + get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr, + get_impl_options.columns, ts, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, &read_cb)) { done = true; - pinnable_val->PinSelf(); + if (get_impl_options.value) { + get_impl_options.value->PinSelf(); + } RecordTick(stats_, MEMTABLE_HIT); } else if ((s.ok() || s.IsMergeInProgress()) && super_version->imm->Get( - lkey, pinnable_val->GetSelf(), /*columns=*/nullptr, ts, &s, - &merge_context, &max_covering_tombstone_seq, read_options, - &read_cb)) { + lkey, + get_impl_options.value ? get_impl_options.value->GetSelf() + : nullptr, + get_impl_options.columns, ts, &s, &merge_context, + &max_covering_tombstone_seq, read_options, &read_cb)) { done = true; - pinnable_val->PinSelf(); + if (get_impl_options.value) { + get_impl_options.value->PinSelf(); + } RecordTick(stats_, MEMTABLE_HIT); } if (!done && !s.ok() && !s.IsMergeInProgress()) { @@ -433,8 +436,8 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options, PERF_TIMER_GUARD(get_from_output_files_time); PinnedIteratorsManager pinned_iters_mgr; super_version->current->Get( - read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s, - &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr, + read_options, lkey, get_impl_options.value, get_impl_options.columns, + ts, &s, &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr, /*value_found*/ nullptr, /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, /*is_blob*/ nullptr, /*do_merge*/ true); @@ -444,7 +447,12 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options, PERF_TIMER_GUARD(get_post_process_time); ReturnAndCleanupSuperVersion(cfd, super_version); RecordTick(stats_, NUMBER_KEYS_READ); - size_t size = pinnable_val->size(); + size_t size = 0; + if (get_impl_options.value) { + size = get_impl_options.value->size(); + } else if (get_impl_options.columns) { + size = get_impl_options.columns->serialized_size(); + } RecordTick(stats_, BYTES_READ, size); RecordTimeToHistogram(stats_, BYTES_PER_READ, size); PERF_COUNTER_ADD(get_read_bytes, size); @@ -452,8 +460,18 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options, return s; } -Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options, +Iterator* DBImplSecondary::NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { + return NewErrorIterator(Status::InvalidArgument( + "Can only call NewIterator with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`")); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; + } if (read_options.managed) { return NewErrorIterator( Status::NotSupported("Managed iterator is not supported anymore.")); @@ -462,16 +480,11 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options, return NewErrorIterator(Status::NotSupported( "ReadTier::kPersistedData is not yet supported in iterators.")); } - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return NewErrorIterator(Status::InvalidArgument( - "Cannot call NewIterator with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`")); - } assert(column_family); if (read_options.timestamp) { - const Status s = FailIfTsMismatchCf( - column_family, *(read_options.timestamp), /*ts_for_read=*/true); + const Status s = + FailIfTsMismatchCf(column_family, *(read_options.timestamp)); if (!s.ok()) { return NewErrorIterator(s); } @@ -495,17 +508,25 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options, Status::NotSupported("snapshot not supported in secondary mode")); } else { SequenceNumber snapshot(kMaxSequenceNumber); - result = NewIteratorImpl(read_options, cfd, snapshot, read_callback); + SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + if (read_options.timestamp && read_options.timestamp->size() > 0) { + const Status s = + FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp)); + if (!s.ok()) { + CleanupSuperVersion(sv); + return NewErrorIterator(s); + } + } + result = NewIteratorImpl(read_options, cfd, sv, snapshot, read_callback); } return result; } ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl( const ReadOptions& read_options, ColumnFamilyData* cfd, - SequenceNumber snapshot, ReadCallback* read_callback, - bool expose_blob_index, bool allow_refresh) { + SuperVersion* super_version, SequenceNumber snapshot, + ReadCallback* read_callback, bool expose_blob_index, bool allow_refresh) { assert(nullptr != cfd); - SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); assert(snapshot == kMaxSequenceNumber); snapshot = versions_->LastSequence(); assert(snapshot != kMaxSequenceNumber); @@ -514,7 +535,7 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl( super_version->current, snapshot, super_version->mutable_cf_options.max_sequential_skip_in_iterations, super_version->version_number, read_callback, this, cfd, - expose_blob_index, read_options.snapshot ? false : allow_refresh); + expose_blob_index, allow_refresh); auto internal_iter = NewInternalIterator( db_iter->GetReadOptions(), cfd, super_version, db_iter->GetArena(), snapshot, /* allow_unprepared_value */ true, db_iter); @@ -523,9 +544,19 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl( } Status DBImplSecondary::NewIterators( - const ReadOptions& read_options, + const ReadOptions& _read_options, const std::vector& column_families, std::vector* iterators) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { + return Status::InvalidArgument( + "Can only call NewIterators with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; + } if (read_options.managed) { return Status::NotSupported("Managed iterator is not supported anymore."); } @@ -533,11 +564,6 @@ Status DBImplSecondary::NewIterators( return Status::NotSupported( "ReadTier::kPersistedData is not yet supported in iterators."); } - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return Status::InvalidArgument( - "Cannot call NewIterators with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - } ReadCallback* read_callback = nullptr; // No read callback provided. if (iterators == nullptr) { return Status::InvalidArgument("iterators not allowed to be nullptr"); @@ -546,8 +572,7 @@ Status DBImplSecondary::NewIterators( if (read_options.timestamp) { for (auto* cf : column_families) { assert(cf); - const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp), - /*ts_for_read=*/true); + const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp)); if (!s.ok()) { return s; } @@ -571,10 +596,28 @@ Status DBImplSecondary::NewIterators( return Status::NotSupported("snapshot not supported in secondary mode"); } else { SequenceNumber read_seq(kMaxSequenceNumber); + autovector> cfd_to_sv; + const bool check_read_ts = + read_options.timestamp && read_options.timestamp->size() > 0; for (auto cfh : column_families) { ColumnFamilyData* cfd = static_cast(cfh)->cfd(); + SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + cfd_to_sv.emplace_back(cfd, sv); + if (check_read_ts) { + const Status s = + FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp)); + if (!s.ok()) { + for (auto prev_entry : cfd_to_sv) { + CleanupSuperVersion(std::get<1>(prev_entry)); + } + return s; + } + } + } + assert(cfd_to_sv.size() == column_families.size()); + for (auto [cfd, sv] : cfd_to_sv) { iterators->push_back( - NewIteratorImpl(read_options, cfd, read_seq, read_callback)); + NewIteratorImpl(read_options, cfd, sv, read_seq, read_callback)); } } return Status::OK(); @@ -842,7 +885,7 @@ Status DBImplSecondary::CompactWithoutInstallation( *mutable_cf_options, mutable_db_options_, 0)); assert(c != nullptr); - c->SetInputVersion(version); + c->FinalizeInputInfo(version); // Create output directory if it's not existed yet std::unique_ptr output_dir; @@ -960,6 +1003,8 @@ Status DB::OpenAndCompact( delete db; if (s.ok()) { return serialization_status; + } else { + serialization_status.PermitUncheckedError(); } return s; } diff --git a/db/db_impl/db_impl_secondary.h b/db/db_impl/db_impl_secondary.h index faaa98721..12a8bbdd7 100644 --- a/db/db_impl/db_impl_secondary.h +++ b/db/db_impl/db_impl_secondary.h @@ -85,8 +85,6 @@ class DBImplSecondary : public DBImpl { bool error_if_data_exists_in_wals, uint64_t* = nullptr, RecoveryContext* recovery_ctx = nullptr) override; - // Implementations of the DB interface. - using DB::Get; // Can return IOError due to files being deleted by the primary. To avoid // IOError in this case, application can coordinate between primary and // secondaries so that primary will not delete files that are currently being @@ -96,16 +94,9 @@ class DBImplSecondary : public DBImpl { // workaround, the secondaries can be opened with `max_open_files=-1` so that // it eagerly keeps all talbe files open and is able to access the contents of // deleted files via prior open fd. - Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* value) override; - - Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* value, - std::string* timestamp) override; - - Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* value, - std::string* timestamp); + using DBImpl::GetImpl; + Status GetImpl(const ReadOptions& options, const Slice& key, + GetImplOptions& get_impl_options) override; using DBImpl::NewIterator; // Operations on the created iterators can return IOError due to files being @@ -117,17 +108,17 @@ class DBImplSecondary : public DBImpl { // deleted. As a partial hacky workaround, the secondaries can be opened with // `max_open_files=-1` so that it eagerly keeps all talbe files open and is // able to access the contents of deleted files via prior open fd. - Iterator* NewIterator(const ReadOptions&, + Iterator* NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) override; ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& read_options, - ColumnFamilyData* cfd, + ColumnFamilyData* cfd, SuperVersion* sv, SequenceNumber snapshot, ReadCallback* read_callback, bool expose_blob_index = false, bool allow_refresh = true); - Status NewIterators(const ReadOptions& options, + Status NewIterators(const ReadOptions& _read_options, const std::vector& column_families, std::vector* iterators) override; @@ -145,6 +136,10 @@ class DBImplSecondary : public DBImpl { const WideColumns& /* columns */) override { return Status::NotSupported("Not supported operation in secondary mode."); } + Status PutEntity(const WriteOptions& /* options */, const Slice& /* key */, + const AttributeGroups& /* attribute_groups */) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } using DBImpl::Merge; Status Merge(const WriteOptions& /*options*/, @@ -324,4 +319,3 @@ class DBImplSecondary : public DBImpl { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/db/db_impl/db_impl_write.cc b/db/db_impl/db_impl_write.cc index 8a1a6ce31..df67ba8c8 100644 --- a/db/db_impl/db_impl_write.cc +++ b/db/db_impl/db_impl_write.cc @@ -30,7 +30,7 @@ Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family, Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family, const Slice& key, const Slice& ts, const Slice& val) { - const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + const Status s = FailIfTsMismatchCf(column_family, ts); if (!s.ok()) { return s; } @@ -48,6 +48,17 @@ Status DBImpl::PutEntity(const WriteOptions& options, return DB::PutEntity(options, column_family, key, columns); } +Status DBImpl::PutEntity(const WriteOptions& options, const Slice& key, + const AttributeGroups& attribute_groups) { + for (const AttributeGroup& ag : attribute_groups) { + const Status s = FailIfCfHasTs(ag.column_family()); + if (!s.ok()) { + return s; + } + } + return DB::PutEntity(options, key, attribute_groups); +} + Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family, const Slice& key, const Slice& val) { const Status s = FailIfCfHasTs(column_family); @@ -64,7 +75,7 @@ Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family, Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family, const Slice& key, const Slice& ts, const Slice& val) { - const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + const Status s = FailIfTsMismatchCf(column_family, ts); if (!s.ok()) { return s; } @@ -83,7 +94,7 @@ Status DBImpl::Delete(const WriteOptions& write_options, Status DBImpl::Delete(const WriteOptions& write_options, ColumnFamilyHandle* column_family, const Slice& key, const Slice& ts) { - const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + const Status s = FailIfTsMismatchCf(column_family, ts); if (!s.ok()) { return s; } @@ -103,7 +114,7 @@ Status DBImpl::SingleDelete(const WriteOptions& write_options, Status DBImpl::SingleDelete(const WriteOptions& write_options, ColumnFamilyHandle* column_family, const Slice& key, const Slice& ts) { - const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + const Status s = FailIfTsMismatchCf(column_family, ts); if (!s.ok()) { return s; } @@ -124,7 +135,7 @@ Status DBImpl::DeleteRange(const WriteOptions& write_options, ColumnFamilyHandle* column_family, const Slice& begin_key, const Slice& end_key, const Slice& ts) { - const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + const Status s = FailIfTsMismatchCf(column_family, ts); if (!s.ok()) { return s; } @@ -403,17 +414,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options, IOStatus io_s; Status pre_release_cb_status; if (status.ok()) { - // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock - // grabs but does not seem thread-safe. - if (tracer_) { - InstrumentedMutexLock lock(&trace_mutex_); - if (tracer_ && tracer_->IsWriteOrderPreserved()) { - for (auto* writer : write_group) { - // TODO: maybe handle the tracing status? - tracer_->Write(writer->batch).PermitUncheckedError(); - } - } - } // Rules for when we can update the memtable concurrently // 1. supported by memtable // 2. Puts are not okay if inplace_update_support @@ -437,15 +437,29 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options, valid_batches += writer->batch_cnt; if (writer->ShouldWriteToMemtable()) { total_count += WriteBatchInternal::Count(writer->batch); + total_byte_size = WriteBatchInternal::AppendedByteSize( + total_byte_size, WriteBatchInternal::ByteSize(writer->batch)); parallel = parallel && !writer->batch->HasMerge(); } - total_byte_size = WriteBatchInternal::AppendedByteSize( - total_byte_size, WriteBatchInternal::ByteSize(writer->batch)); if (writer->pre_release_callback) { pre_release_callback_cnt++; } } } + // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock + // grabs but does not seem thread-safe. + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_ && tracer_->IsWriteOrderPreserved()) { + for (auto* writer : write_group) { + if (writer->CallbackFailed()) { + continue; + } + // TODO: maybe handle the tracing status? + tracer_->Write(writer->batch).PermitUncheckedError(); + } + } + } // Note about seq_per_batch_: either disableWAL is set for the entire write // group or not. In either case we inc seq for each write batch with no // failed callback. This means that there could be a batch with @@ -706,11 +720,11 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options, if (writer->ShouldWriteToMemtable()) { writer->sequence = next_sequence; size_t count = WriteBatchInternal::Count(writer->batch); + total_byte_size = WriteBatchInternal::AppendedByteSize( + total_byte_size, WriteBatchInternal::ByteSize(writer->batch)); next_sequence += count; total_count += count; } - total_byte_size = WriteBatchInternal::AppendedByteSize( - total_byte_size, WriteBatchInternal::ByteSize(writer->batch)); } } if (w.disable_wal) { @@ -1811,11 +1825,9 @@ uint64_t DBImpl::GetMaxTotalWalSize() const { Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, const WriteOptions& write_options) { mutex_.AssertHeld(); - uint64_t time_delayed = 0; + uint64_t start_time = 0; bool delayed = false; { - StopWatch sw(immutable_db_options_.clock, stats_, WRITE_STALL, - Histograms::HISTOGRAM_ENUM_MAX, &time_delayed); // To avoid parallel timed delays (bad throttling), only support them // on the primary write queue. uint64_t delay; @@ -1827,6 +1839,8 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, delay = 0; } TEST_SYNC_POINT("DBImpl::DelayWrite:Start"); + start_time = immutable_db_options_.clock->NowMicros(); + if (delay > 0) { if (write_options.no_slowdown) { return Status::Incomplete("Write stall"); @@ -1843,7 +1857,7 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, // (slightly longer because WriteController minimum delay is 1ms, in // case of sleep imprecision, rounding, etc.) const uint64_t kDelayInterval = 1001; - uint64_t stall_end = sw.start_time() + delay; + uint64_t stall_end = start_time + delay; while (write_controller_.NeedsDelay()) { if (immutable_db_options_.clock->NowMicros() >= stall_end) { // We already delayed this write `delay` microseconds @@ -1858,11 +1872,11 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, write_thread.EndWriteStall(); } - // Don't wait if there's a background error, even if its a soft error. We - // might wait here indefinitely as the background compaction may never - // finish successfully, resulting in the stall condition lasting - // indefinitely - while (error_handler_.GetBGError().ok() && write_controller_.IsStopped() && + // Don't wait if there's a background error that is not pending recovery + // since recovery might never be attempted. + while ((error_handler_.GetBGError().ok() || + error_handler_.IsRecoveryInProgress()) && + write_controller_.IsStopped() && !shutting_down_.load(std::memory_order_relaxed)) { if (write_options.no_slowdown) { return Status::Incomplete("Write stall"); @@ -1884,9 +1898,11 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, } assert(!delayed || !write_options.no_slowdown); if (delayed) { + auto time_delayed = immutable_db_options_.clock->NowMicros() - start_time; default_cf_internal_stats_->AddDBStats( InternalStats::kIntStatsWriteStallMicros, time_delayed); RecordTick(stats_, STALL_MICROS, time_delayed); + RecordInHistogram(stats_, WRITE_STALL, time_delayed); } // If DB is not in read-only mode and write_controller is not stopping @@ -1953,9 +1969,13 @@ Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options, // a chance to run. Now we guarantee we are still slowly making // progress. PERF_TIMER_GUARD(write_delay_time); - write_controller_.low_pri_rate_limiter()->Request( - my_batch->GetDataSize(), Env::IO_HIGH, nullptr /* stats */, - RateLimiter::OpType::kWrite); + auto data_size = my_batch->GetDataSize(); + while (data_size > 0) { + size_t allowed = write_controller_.low_pri_rate_limiter()->RequestToken( + data_size, 0 /* alignment */, Env::IO_HIGH, nullptr /* stats */, + RateLimiter::OpType::kWrite); + data_size -= allowed; + } } } return Status::OK(); @@ -2378,6 +2398,22 @@ Status DB::PutEntity(const WriteOptions& options, return Write(options, &batch); } +Status DB::PutEntity(const WriteOptions& options, const Slice& key, + const AttributeGroups& attribute_groups) { + ColumnFamilyHandle* default_cf = DefaultColumnFamily(); + assert(default_cf); + const Comparator* const default_cf_ucmp = default_cf->GetComparator(); + assert(default_cf_ucmp); + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + options.protection_bytes_per_key, + default_cf_ucmp->timestamp_size()); + const Status s = batch.PutEntity(key, attribute_groups); + if (!s.ok()) { + return s; + } + return Write(options, &batch); +} + Status DB::Delete(const WriteOptions& opt, ColumnFamilyHandle* column_family, const Slice& key) { WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, diff --git a/db/db_info_dumper.cc b/db/db_info_dumper.cc index be8d5bee1..7dd647955 100644 --- a/db/db_info_dumper.cc +++ b/db/db_info_dumper.cc @@ -34,6 +34,12 @@ void DumpDBFileSummary(const ImmutableDBOptions& options, std::string file_info, wal_info; Header(options.info_log, "DB SUMMARY\n"); + { + std::string hostname; + if (env->GetHostNameString(&hostname).ok()) { + Header(options.info_log, "Host name (Env): %s\n", hostname.c_str()); + } + } Header(options.info_log, "DB Session ID: %s\n", session_id.c_str()); Status s; diff --git a/db/db_iter.cc b/db/db_iter.cc index 3d980c878..507bb2577 100644 --- a/db/db_iter.cc +++ b/db/db_iter.cc @@ -18,6 +18,7 @@ #include "db/merge_helper.h" #include "db/pinned_iterators_manager.h" #include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "file/filename.h" #include "logging/logging.h" #include "memory/arena.h" @@ -77,6 +78,7 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options, expose_blob_index_(expose_blob_index), is_blob_(false), arena_mode_(arena_mode), + io_activity_(read_options.io_activity), db_impl_(db_impl), cfd_(cfd), timestamp_ub_(read_options.timestamp), @@ -111,6 +113,9 @@ Status DBIter::GetProperty(std::string prop_name, std::string* prop) { } else if (prop_name == "rocksdb.iterator.internal-key") { *prop = saved_key_.GetUserKey().ToString(); return Status::OK(); + } else if (prop_name == "rocksdb.iterator.write-time") { + // TODO(yuzhangyu): implement return the actual write time. + return Status::NotSupported("write time property is under construction"); } return Status::InvalidArgument("Unidentified property."); } @@ -196,12 +201,11 @@ bool DBIter::SetBlobValueIfNeeded(const Slice& user_key, // TODO: consider moving ReadOptions from ArenaWrappedDBIter to DBIter to // avoid having to copy options back and forth. - // TODO: plumb Env::IOActivity ReadOptions read_options; read_options.read_tier = read_tier_; read_options.fill_cache = fill_cache_; read_options.verify_checksums = verify_checksums_; - + read_options.io_activity = io_activity_; constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; constexpr uint64_t* bytes_read = nullptr; @@ -230,14 +234,38 @@ bool DBIter::SetValueAndColumnsFromEntity(Slice slice) { return false; } - if (!wide_columns_.empty() && - wide_columns_[0].name() == kDefaultWideColumnName) { - value_ = wide_columns_[0].value(); + if (WideColumnsHelper::HasDefaultColumn(wide_columns_)) { + value_ = WideColumnsHelper::GetDefaultColumn(wide_columns_); } return true; } +bool DBIter::SetValueAndColumnsFromMergeResult(const Status& merge_status, + ValueType result_type) { + if (!merge_status.ok()) { + valid_ = false; + status_ = merge_status; + return false; + } + + if (result_type == kTypeWideColumnEntity) { + if (!SetValueAndColumnsFromEntity(saved_value_)) { + assert(!valid_); + return false; + } + + valid_ = true; + return true; + } + + assert(result_type == kTypeValue); + SetValueAndColumnsFromPlain(pinned_value_.data() ? pinned_value_ + : saved_value_); + valid_ = true; + return true; +} + // PRE: saved_key_ has the current user key if skipping_saved_key // POST: saved_key_ should have the next user key if valid_, // if the current entry is a result of merge @@ -554,8 +582,7 @@ bool DBIter::MergeValuesNewToOld() { if (kTypeValue == ikey.type) { // hit a put, merge the put value with operands and store the // final result in saved_value_. We are done! - const Slice val = iter_.value(); - if (!Merge(&val, ikey.user_key)) { + if (!MergeWithPlainBaseValue(iter_.value(), ikey.user_key)) { return false; } // iter_ is positioned after put @@ -584,7 +611,7 @@ bool DBIter::MergeValuesNewToOld() { return false; } valid_ = true; - if (!Merge(&blob_value_, ikey.user_key)) { + if (!MergeWithPlainBaseValue(blob_value_, ikey.user_key)) { return false; } @@ -598,7 +625,7 @@ bool DBIter::MergeValuesNewToOld() { } return true; } else if (kTypeWideColumnEntity == ikey.type) { - if (!MergeEntity(iter_.value(), ikey.user_key)) { + if (!MergeWithWideColumnBaseValue(iter_.value(), ikey.user_key)) { return false; } @@ -628,7 +655,7 @@ bool DBIter::MergeValuesNewToOld() { // a deletion marker. // feed null as the existing value to the merge operator, such that // client can differentiate this scenario and do things accordingly. - if (!Merge(nullptr, saved_key_.GetUserKey())) { + if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) { return false; } assert(status_.ok()); @@ -979,7 +1006,7 @@ bool DBIter::FindValueForCurrentKey() { if (last_not_merge_type == kTypeDeletion || last_not_merge_type == kTypeSingleDeletion || last_not_merge_type == kTypeDeletionWithTimestamp) { - if (!Merge(nullptr, saved_key_.GetUserKey())) { + if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) { return false; } return true; @@ -994,7 +1021,7 @@ bool DBIter::FindValueForCurrentKey() { return false; } valid_ = true; - if (!Merge(&blob_value_, saved_key_.GetUserKey())) { + if (!MergeWithPlainBaseValue(blob_value_, saved_key_.GetUserKey())) { return false; } @@ -1002,14 +1029,15 @@ bool DBIter::FindValueForCurrentKey() { return true; } else if (last_not_merge_type == kTypeWideColumnEntity) { - if (!MergeEntity(pinned_value_, saved_key_.GetUserKey())) { + if (!MergeWithWideColumnBaseValue(pinned_value_, + saved_key_.GetUserKey())) { return false; } return true; } else { assert(last_not_merge_type == kTypeValue); - if (!Merge(&pinned_value_, saved_key_.GetUserKey())) { + if (!MergeWithPlainBaseValue(pinned_value_, saved_key_.GetUserKey())) { return false; } return true; @@ -1185,8 +1213,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() { } if (ikey.type == kTypeValue) { - const Slice val = iter_.value(); - if (!Merge(&val, saved_key_.GetUserKey())) { + if (!MergeWithPlainBaseValue(iter_.value(), saved_key_.GetUserKey())) { return false; } return true; @@ -1205,7 +1232,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() { return false; } valid_ = true; - if (!Merge(&blob_value_, saved_key_.GetUserKey())) { + if (!MergeWithPlainBaseValue(blob_value_, saved_key_.GetUserKey())) { return false; } @@ -1213,7 +1240,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() { return true; } else if (ikey.type == kTypeWideColumnEntity) { - if (!MergeEntity(iter_.value(), saved_key_.GetUserKey())) { + if (!MergeWithWideColumnBaseValue(iter_.value(), + saved_key_.GetUserKey())) { return false; } @@ -1227,7 +1255,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() { } } - if (!Merge(nullptr, saved_key_.GetUserKey())) { + if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) { return false; } @@ -1250,47 +1278,42 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() { return true; } -bool DBIter::Merge(const Slice* val, const Slice& user_key) { +bool DBIter::MergeWithNoBaseValue(const Slice& user_key) { // `op_failure_scope` (an output parameter) is not provided (set to nullptr) // since a failure must be propagated regardless of its value. - Status s = MergeHelper::TimedFullMerge( - merge_operator_, user_key, val, merge_context_.GetOperands(), - &saved_value_, logger_, statistics_, clock_, &pinned_value_, - /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - if (!s.ok()) { - valid_ = false; - status_ = s; - return false; - } - - SetValueAndColumnsFromPlain(pinned_value_.data() ? pinned_value_ - : saved_value_); - - valid_ = true; - return true; + ValueType result_type; + const Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key, MergeHelper::kNoBaseValue, + merge_context_.GetOperands(), logger_, statistics_, clock_, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + &saved_value_, &pinned_value_, &result_type); + return SetValueAndColumnsFromMergeResult(s, result_type); } -bool DBIter::MergeEntity(const Slice& entity, const Slice& user_key) { +bool DBIter::MergeWithPlainBaseValue(const Slice& value, + const Slice& user_key) { // `op_failure_scope` (an output parameter) is not provided (set to nullptr) // since a failure must be propagated regardless of its value. - Status s = MergeHelper::TimedFullMergeWithEntity( - merge_operator_, user_key, entity, merge_context_.GetOperands(), - &saved_value_, logger_, statistics_, clock_, - /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - if (!s.ok()) { - valid_ = false; - status_ = s; - return false; - } - - if (!SetValueAndColumnsFromEntity(saved_value_)) { - return false; - } + ValueType result_type; + const Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key, MergeHelper::kPlainBaseValue, value, + merge_context_.GetOperands(), logger_, statistics_, clock_, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + &saved_value_, &pinned_value_, &result_type); + return SetValueAndColumnsFromMergeResult(s, result_type); +} - valid_ = true; - return true; +bool DBIter::MergeWithWideColumnBaseValue(const Slice& entity, + const Slice& user_key) { + // `op_failure_scope` (an output parameter) is not provided (set to nullptr) + // since a failure must be propagated regardless of its value. + ValueType result_type; + const Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key, MergeHelper::kWideBaseValue, entity, + merge_context_.GetOperands(), logger_, statistics_, clock_, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + &saved_value_, &pinned_value_, &result_type); + return SetValueAndColumnsFromMergeResult(s, result_type); } // Move backwards until the key smaller than saved_key_. diff --git a/db/db_iter.h b/db/db_iter.h index e1663bb7e..5022405c3 100644 --- a/db/db_iter.h +++ b/db/db_iter.h @@ -209,6 +209,7 @@ class DBIter final : public Iterator { if (read_callback_) { read_callback_->Refresh(s); } + iter_.SetRangeDelReadSeqno(s); } void set_valid(bool v) { valid_ = v; } @@ -312,14 +313,20 @@ class DBIter final : public Iterator { bool SetValueAndColumnsFromEntity(Slice slice); + bool SetValueAndColumnsFromMergeResult(const Status& merge_status, + ValueType result_type); + void ResetValueAndColumns() { value_.clear(); wide_columns_.clear(); } + // The following methods perform the actual merge operation for the + // no base value/plain base value/wide-column base value cases. // If user-defined timestamp is enabled, `user_key` includes timestamp. - bool Merge(const Slice* val, const Slice& user_key); - bool MergeEntity(const Slice& entity, const Slice& user_key); + bool MergeWithNoBaseValue(const Slice& user_key); + bool MergeWithPlainBaseValue(const Slice& value, const Slice& user_key); + bool MergeWithWideColumnBaseValue(const Slice& entity, const Slice& user_key); const SliceTransform* prefix_extractor_; Env* const env_; @@ -384,6 +391,7 @@ class DBIter final : public Iterator { bool expose_blob_index_; bool is_blob_; bool arena_mode_; + const Env::IOActivity io_activity_; // List of operands for merge operator. MergeContext merge_context_; LocalStatistics local_stats_; diff --git a/db/db_iter_test.cc b/db/db_iter_test.cc index 65290bfad..6fd446970 100644 --- a/db/db_iter_test.cc +++ b/db/db_iter_test.cc @@ -275,6 +275,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { db_iter->Next(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } // Test to check the SeekToLast() with iterate_upper_bound not set { @@ -1415,6 +1416,7 @@ TEST_F(DBIteratorTest, DBIterator1) { ASSERT_EQ(db_iter->key().ToString(), "b"); db_iter->Next(); ASSERT_FALSE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); } TEST_F(DBIteratorTest, DBIterator2) { @@ -1528,6 +1530,7 @@ TEST_F(DBIteratorTest, DBIterator5) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1552,6 +1555,7 @@ TEST_F(DBIteratorTest, DBIterator5) { ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1576,6 +1580,7 @@ TEST_F(DBIteratorTest, DBIterator5) { ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1600,6 +1605,7 @@ TEST_F(DBIteratorTest, DBIterator5) { ASSERT_EQ(db_iter->value().ToString(), "put_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1624,6 +1630,7 @@ TEST_F(DBIteratorTest, DBIterator5) { ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1648,6 +1655,7 @@ TEST_F(DBIteratorTest, DBIterator5) { ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1672,6 +1680,7 @@ TEST_F(DBIteratorTest, DBIterator5) { ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5,merge_6"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1726,6 +1735,7 @@ TEST_F(DBIteratorTest, DBIterator6) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1750,6 +1760,7 @@ TEST_F(DBIteratorTest, DBIterator6) { ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1774,6 +1785,7 @@ TEST_F(DBIteratorTest, DBIterator6) { ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1794,6 +1806,7 @@ TEST_F(DBIteratorTest, DBIterator6) { nullptr /* read_callback */)); db_iter->SeekToLast(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1818,6 +1831,7 @@ TEST_F(DBIteratorTest, DBIterator6) { ASSERT_EQ(db_iter->value().ToString(), "merge_4"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1842,6 +1856,7 @@ TEST_F(DBIteratorTest, DBIterator6) { ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1866,6 +1881,7 @@ TEST_F(DBIteratorTest, DBIterator6) { ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5,merge_6"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } } @@ -1910,6 +1926,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1952,6 +1969,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -1994,6 +2012,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -2041,6 +2060,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -2089,6 +2109,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -2131,6 +2152,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -2179,6 +2201,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -2228,6 +2251,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } { @@ -2271,6 +2295,7 @@ TEST_F(DBIteratorTest, DBIterator7) { ASSERT_EQ(db_iter->value().ToString(), "merge_1"); db_iter->Prev(); ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); } } @@ -2440,6 +2465,7 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) { ASSERT_EQ(db_iter->value().ToString(), "2"); db_iter->Next(); ASSERT_FALSE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); } TEST_F(DBIteratorTest, DBIterator11) { @@ -2469,6 +2495,7 @@ TEST_F(DBIteratorTest, DBIterator11) { ASSERT_EQ(db_iter->key().ToString(), "b"); db_iter->Next(); ASSERT_FALSE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); } TEST_F(DBIteratorTest, DBIterator12) { @@ -2497,6 +2524,7 @@ TEST_F(DBIteratorTest, DBIterator12) { ASSERT_EQ(db_iter->value().ToString(), "1"); db_iter->Prev(); ASSERT_FALSE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); } TEST_F(DBIteratorTest, DBIterator13) { @@ -2635,6 +2663,7 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIterator1) { ASSERT_EQ(db_iter_->value().ToString(), "3"); db_iter_->Next(); ASSERT_FALSE(db_iter_->Valid()); + ASSERT_OK(db_iter_->status()); } TEST_F(DBIterWithMergeIterTest, InnerMergeIterator2) { diff --git a/db/db_iterator_test.cc b/db/db_iterator_test.cc index 862377b6d..29c39f6ad 100644 --- a/db/db_iterator_test.cc +++ b/db/db_iterator_test.cc @@ -107,7 +107,10 @@ class DBIteratorTest : public DBIteratorBaseTest, read_callbacks_.push_back( std::unique_ptr(read_callback)); } - return dbfull()->NewIteratorImpl(read_options, cfd, seq, read_callback); + DBImpl* db_impl = dbfull(); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl); + return db_impl->NewIteratorImpl(read_options, cfd, super_version, seq, + read_callback); } private: @@ -180,6 +183,7 @@ TEST_P(DBIteratorTest, NonBlockingIteration) { ASSERT_OK(iter->status()); count++; } + ASSERT_OK(iter->status()); ASSERT_EQ(count, 1); delete iter; @@ -214,6 +218,7 @@ TEST_P(DBIteratorTest, NonBlockingIteration) { ASSERT_OK(iter->status()); count++; } + ASSERT_OK(iter->status()); ASSERT_EQ(count, 1); ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS)); ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD)); @@ -867,6 +872,7 @@ TEST_P(DBIteratorTest, IterWithSnapshot) { } } db_->ReleaseSnapshot(snapshot); + ASSERT_OK(iter->status()); delete iter; } while (ChangeOptions()); } @@ -1211,6 +1217,7 @@ TEST_P(DBIteratorTest, DBIteratorBoundOptimizationTest) { iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); ASSERT_EQ(upper_bound_hits, 1); } } @@ -1335,6 +1342,7 @@ TEST_P(DBIteratorTest, IndexWithFirstKey) { iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); EXPECT_EQ(7, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); } @@ -1576,6 +1584,7 @@ class DBIteratorTestForPinnedData : public DBIteratorTest { ASSERT_EQ("1", prop_value); all_keys.push_back(iter->key()); } + ASSERT_OK(iter->status()); ASSERT_EQ(all_keys.size(), true_data.size()); // Verify that all keys slices are valid (backward) @@ -1679,7 +1688,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorMultipleFiles) { ASSERT_EQ(kv.first, data_iter->first); ASSERT_EQ(kv.second, data_iter->second); } - + ASSERT_OK(iter->status()); delete iter; } @@ -1725,6 +1734,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorMergeOperator) { ASSERT_EQ("1", prop_value); results.emplace_back(iter->key(), iter->value().ToString()); } + ASSERT_OK(iter->status()); ASSERT_EQ(results.size(), 1000); for (size_t i = 0; i < results.size(); i++) { @@ -1782,6 +1792,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorReadAfterUpdate) { ASSERT_EQ("1", prop_value); results.emplace_back(iter->key(), iter->value().ToString()); } + ASSERT_OK(iter->status()); auto data_iter = true_data.begin(); for (size_t i = 0; i < results.size(); i++, data_iter++) { @@ -2076,6 +2087,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) { ASSERT_EQ(iter->value().ToString(), data_iter->second); data_iter++; } + ASSERT_OK(iter->status()); ASSERT_EQ(data_iter, true_data.rend()); delete iter; @@ -2133,6 +2145,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) { entries_right++; data_iter++; } + ASSERT_OK(iter->status()); ASSERT_EQ(data_iter, true_data.rend()); delete iter; @@ -2172,6 +2185,7 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) { total_next++; if (!iter->Valid()) { + EXPECT_OK(iter->status()); break; } total_next_found++; @@ -2199,6 +2213,7 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) { total_prev++; if (!iter->Valid()) { + EXPECT_OK(iter->status()); break; } total_prev_found++; @@ -2413,37 +2428,98 @@ TEST_P(DBIteratorTest, Refresh) { ASSERT_EQ(iter->key().compare(Slice("x")), 0); iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); iter.reset(); } TEST_P(DBIteratorTest, RefreshWithSnapshot) { - ASSERT_OK(Put("x", "y")); + // L1 file, uses LevelIterator internally + ASSERT_OK(Put(Key(0), "val0")); + ASSERT_OK(Put(Key(5), "val5")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + // L0 file, uses table iterator internally + ASSERT_OK(Put(Key(1), "val1")); + ASSERT_OK(Put(Key(4), "val4")); + ASSERT_OK(Flush()); + + // Memtable + ASSERT_OK(Put(Key(2), "val2")); + ASSERT_OK(Put(Key(3), "val3")); const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(Put(Key(2), "new val")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(4), + Key(7))); + const Snapshot* snapshot2 = db_->GetSnapshot(); + + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ReadOptions options; options.snapshot = snapshot; Iterator* iter = NewIterator(options); + ASSERT_OK(Put(Key(6), "val6")); ASSERT_OK(iter->status()); - iter->Seek(Slice("a")); - ASSERT_TRUE(iter->Valid()); - ASSERT_EQ(iter->key().compare(Slice("x")), 0); - iter->Next(); - ASSERT_FALSE(iter->Valid()); + auto verify_iter = [&](int start, int end, bool new_key2 = false) { + for (int i = start; i < end; ++i) { + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(i)); + if (i == 2 && new_key2) { + ASSERT_EQ(iter->value(), "new val"); + } else { + ASSERT_EQ(iter->value(), "val" + std::to_string(i)); + } + iter->Next(); + } + }; - ASSERT_OK(Put("c", "d")); + for (int j = 0; j < 2; j++) { + iter->Seek(Key(1)); + verify_iter(1, 3); + // Refresh to same snapshot + ASSERT_OK(iter->Refresh(snapshot)); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + iter->Seek(Key(3)); + verify_iter(3, 6); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + + // Refresh to a newer snapshot + ASSERT_OK(iter->Refresh(snapshot2)); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + iter->SeekToFirst(); + verify_iter(0, 4, /*new_key2=*/true); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + + // Refresh to an older snapshot + ASSERT_OK(iter->Refresh(snapshot)); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + iter->Seek(Key(3)); + verify_iter(3, 6); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + + // Refresh to no snapshot + ASSERT_OK(iter->Refresh()); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + iter->Seek(Key(2)); + verify_iter(2, 4, /*new_key2=*/true); + verify_iter(6, 7); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + + // Change LSM shape, new SuperVersion is created. + ASSERT_OK(Flush()); - iter->Seek(Slice("a")); - ASSERT_TRUE(iter->Valid()); - ASSERT_EQ(iter->key().compare(Slice("x")), 0); - iter->Next(); - ASSERT_FALSE(iter->Valid()); + // Refresh back to original snapshot + ASSERT_OK(iter->Refresh(snapshot)); + } - ASSERT_OK(iter->status()); - Status s = iter->Refresh(); - ASSERT_TRUE(s.IsNotSupported()); - db_->ReleaseSnapshot(snapshot); delete iter; + db_->ReleaseSnapshot(snapshot); + db_->ReleaseSnapshot(snapshot2); + ASSERT_OK(db_->Close()); } TEST_P(DBIteratorTest, CreationFailure) { @@ -2536,6 +2612,7 @@ TEST_P(DBIteratorTest, TableFilter) { ASSERT_EQ(IterStatus(iter), "f->6"); iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); ASSERT_TRUE(unseen.empty()); delete iter; } @@ -2558,6 +2635,7 @@ TEST_P(DBIteratorTest, TableFilter) { ASSERT_EQ(IterStatus(iter), "f->6"); iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); delete iter; } } @@ -2642,6 +2720,7 @@ TEST_P(DBIteratorTest, SkipStatistics) { ASSERT_OK(iter->status()); count++; } + ASSERT_OK(iter->status()); ASSERT_EQ(count, 3); delete iter; skip_count += 8; // Same as above, but in reverse order @@ -2677,6 +2756,7 @@ TEST_P(DBIteratorTest, SkipStatistics) { ASSERT_OK(iter->status()); count++; } + ASSERT_OK(iter->status()); ASSERT_EQ(count, 2); delete iter; // 3 deletes + 3 original keys + lower sequence of "a" @@ -3130,8 +3210,10 @@ TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) { static_cast_with_check(db_->DefaultColumnFamily()) ->cfd(); // The iterator are suppose to see data before seq1. - Iterator* iter = - dbfull()->NewIteratorImpl(ReadOptions(), cfd, seq2, &callback1); + DBImpl* db_impl = dbfull(); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl); + Iterator* iter = db_impl->NewIteratorImpl(ReadOptions(), cfd, super_version, + seq2, &callback1); // Seek // The latest value of "foo" before seq1 is "v3" @@ -3209,7 +3291,9 @@ TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) { SequenceNumber seq4 = db_->GetLatestSequenceNumber(); // The iterator is suppose to see data before seq3. - iter = dbfull()->NewIteratorImpl(ReadOptions(), cfd, seq4, &callback2); + super_version = cfd->GetReferencedSuperVersion(db_impl); + iter = db_impl->NewIteratorImpl(ReadOptions(), cfd, super_version, seq4, + &callback2); // Seek to "z", which is visible. iter->Seek("z"); ASSERT_TRUE(iter->Valid()); @@ -3255,6 +3339,7 @@ TEST_F(DBIteratorTest, BackwardIterationOnInplaceUpdateMemtable) { for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { ++count; } + ASSERT_OK(iter->status()); ASSERT_EQ(kNumKeys, count); } @@ -3295,6 +3380,176 @@ TEST_F(DBIteratorTest, IteratorRefreshReturnSV) { Close(); } +TEST_F(DBIteratorTest, ErrorWhenReadFile) { + // This is to test a bug that is fixed in + // https://github.com/facebook/rocksdb/pull/11782. + // + // Ingest error when reading from a file, and + // see if Iterator handles it correctly. + Options opts = CurrentOptions(); + opts.num_levels = 7; + opts.compression = kNoCompression; + BlockBasedTableOptions bbto; + // Always do I/O + bbto.no_block_cache = true; + opts.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(opts); + + // Set up LSM + // L5: F1 [key0, key99], F2 [key100, key199] + // L6: F3 [key50, key149] + Random rnd(301); + const int kValLen = 100; + for (int i = 50; i < 150; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValLen))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + + std::vector values; + for (int i = 0; i < 100; ++i) { + values.emplace_back(rnd.RandomString(kValLen)); + ASSERT_OK(Put(Key(i), values.back())); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + + for (int i = 100; i < 200; ++i) { + values.emplace_back(rnd.RandomString(kValLen)); + ASSERT_OK(Put(Key(i), values.back())); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + + ASSERT_EQ(2, NumTableFilesAtLevel(5)); + ASSERT_EQ(1, NumTableFilesAtLevel(6)); + + std::vector files; + db_->GetLiveFilesMetaData(&files); + // Get file names for F1, F2 and F3. + // These are file names, not full paths. + std::string f1, f2, f3; + for (auto& file_meta : files) { + if (file_meta.level == 6) { + f3 = file_meta.name; + } else { + if (file_meta.smallestkey == Key(0)) { + f1 = file_meta.name; + } else { + f2 = file_meta.name; + } + } + } + ASSERT_TRUE(!f1.empty()); + ASSERT_TRUE(!f2.empty()); + ASSERT_TRUE(!f3.empty()); + + std::string error_file; + SyncPoint::GetInstance()->SetCallBack( + "RandomAccessFileReader::Read::BeforeReturn", + [&error_file](void* io_s_ptr) { + auto p = + reinterpret_cast*>(io_s_ptr); + if (p->first->find(error_file) != std::string::npos) { + *p->second = IOStatus::IOError(); + p->second->SetRetryable(true); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + // Error reading F1 + error_file = f1; + std::unique_ptr iter{db_->NewIterator(ReadOptions())}; + iter->SeekToFirst(); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + // This does not require reading the first block. + iter->Seek(Key(90)); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value(), values[90]); + // iter has ok status before this Seek. + iter->Seek(Key(1)); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + + // Error reading F2 + error_file = f2; + iter.reset(db_->NewIterator(ReadOptions())); + iter->Seek(Key(99)); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value(), values[99]); + // Need to read from F2. + iter->Next(); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + iter->Seek(Key(190)); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value(), values[190]); + // Seek for first key of F2. + iter->Seek(Key(100)); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + iter->SeekToLast(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value(), values[199]); + // SeekForPrev for first key of F2. + iter->SeekForPrev(Key(100)); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + // Does not read first block (offset 0). + iter->SeekForPrev(Key(98)); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value(), values[98]); + + // Error reading F3 + error_file = f3; + iter.reset(db_->NewIterator(ReadOptions())); + iter->SeekToFirst(); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + iter->Seek(Key(50)); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + iter->SeekForPrev(Key(50)); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + // Does not read file 3 + iter->Seek(Key(150)); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value(), values[150]); + + // Test when file read error occurs during Prev(). + // This requires returning an error when reading near the end of a file + // instead of offset 0. + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "RandomAccessFileReader::Read::AnyOffset", [&f1](void* pair_ptr) { + auto p = + reinterpret_cast*>(pair_ptr); + if (p->first->find(f1) != std::string::npos) { + *p->second = IOStatus::IOError(); + p->second->SetRetryable(true); + } + }); + iter->SeekForPrev(Key(101)); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value(), values[101]); + // DBIter will not stop at Key(100) since it needs + // to make sure the key it returns has the max sequence number for Key(100). + // So it will call MergingIterator::Prev() which will read F1. + iter->Prev(); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + SyncPoint::GetInstance()->DisableProcessing(); + iter->Reset(); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/db_log_iter_test.cc b/db/db_log_iter_test.cc index 4c9434586..87313971a 100644 --- a/db/db_log_iter_test.cc +++ b/db/db_log_iter_test.cc @@ -145,6 +145,41 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) { } while (ChangeCompactOptions()); } } + +TEST_F(DBTestXactLogIterator, TransactionLogIteratorCheckWhenArchive) { + do { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + ColumnFamilyHandle* cf; + auto s = dbfull()->CreateColumnFamily(ColumnFamilyOptions(), "CF", &cf); + ASSERT_TRUE(s.ok()); + + ASSERT_OK(dbfull()->Put(WriteOptions(), cf, "key1", DummyString(1024))); + + ASSERT_OK(dbfull()->Put(WriteOptions(), "key2", DummyString(1024))); + + ASSERT_OK(dbfull()->Flush(FlushOptions())); + + ASSERT_OK(dbfull()->Put(WriteOptions(), "key3", DummyString(1024))); + + ASSERT_OK(dbfull()->Flush(FlushOptions())); + + ASSERT_OK(dbfull()->Put(WriteOptions(), "key4", DummyString(1024))); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WalManager::PurgeObsoleteFiles:1", [&](void*) { + auto iter = OpenTransactionLogIter(0); + ExpectRecords(4, iter); + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->Flush(FlushOptions(), cf)); + + delete cf; + } while (ChangeCompactOptions()); +} #endif TEST_F(DBTestXactLogIterator, TransactionLogIteratorStallAtLastRecord) { @@ -201,7 +236,7 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) { ASSERT_OK(test::TruncateFile(env_, logfile_path, wal_files.front()->SizeFileBytes() / 2)); - ASSERT_OK(db_->EnableFileDeletions()); + ASSERT_OK(db_->EnableFileDeletions(/*force=*/false)); // Insert a new entry to a new log file ASSERT_OK(Put("key1025", DummyString(10))); diff --git a/db/db_merge_operator_test.cc b/db/db_merge_operator_test.cc index aa1253a0b..e82e0cbf0 100644 --- a/db/db_merge_operator_test.cc +++ b/db/db_merge_operator_test.cc @@ -6,9 +6,12 @@ #include #include "db/db_test_util.h" +#include "db/dbformat.h" #include "db/forward_iterator.h" #include "port/stack_trace.h" #include "rocksdb/merge_operator.h" +#include "rocksdb/snapshot.h" +#include "rocksdb/utilities/debug.h" #include "util/random.h" #include "utilities/merge_operators.h" #include "utilities/merge_operators/string_append/stringappend2.h" @@ -202,7 +205,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) { VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}}); } - TEST_F(DBMergeOperatorTest, MergeOperatorFailsWithMustMerge) { // This is like a mini-stress test dedicated to `OpFailureScope::kMustMerge`. // Some or most of it might be deleted upon adding that option to the actual @@ -358,6 +360,98 @@ TEST_F(DBMergeOperatorTest, MergeOperatorFailsWithMustMerge) { } } +TEST_F(DBMergeOperatorTest, MergeOperandThresholdExceeded) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.merge_operator = MergeOperators::CreatePutOperator(); + options.env = env_; + Reopen(options); + + std::vector keys{"foo", "bar", "baz"}; + + // Write base values. + for (const auto& key : keys) { + ASSERT_OK(Put(key, key.ToString() + "0")); + } + + // Write merge operands. Note that the first key has 1 merge operand, the + // second one has 2 merge operands, and the third one has 3 merge operands. + // Also, we'll take some snapshots to make sure the merge operands are + // preserved during flush. + std::vector snapshots; + snapshots.reserve(3); + + for (size_t i = 0; i < keys.size(); ++i) { + snapshots.emplace_back(db_); + + const std::string suffix = std::to_string(i + 1); + + for (size_t j = i; j < keys.size(); ++j) { + ASSERT_OK(Merge(keys[j], keys[j].ToString() + suffix)); + } + } + + // Verify the results and status codes of various types of point lookups. + auto verify = [&](const std::optional& threshold) { + ReadOptions read_options; + read_options.merge_operand_count_threshold = threshold; + + // Check Get() + { + for (size_t i = 0; i < keys.size(); ++i) { + PinnableSlice value; + const Status status = + db_->Get(read_options, db_->DefaultColumnFamily(), keys[i], &value); + ASSERT_OK(status); + ASSERT_EQ(status.IsOkMergeOperandThresholdExceeded(), + threshold.has_value() && i + 1 > threshold.value()); + ASSERT_EQ(value, keys[i].ToString() + std::to_string(i + 1)); + } + } + + // Check old-style MultiGet() + { + std::vector values; + std::vector statuses = db_->MultiGet(read_options, keys, &values); + + for (size_t i = 0; i < keys.size(); ++i) { + ASSERT_OK(statuses[i]); + ASSERT_EQ(statuses[i].IsOkMergeOperandThresholdExceeded(), + threshold.has_value() && i + 1 > threshold.value()); + ASSERT_EQ(values[i], keys[i].ToString() + std::to_string(i + 1)); + } + } + + // Check batched MultiGet() + { + std::vector values(keys.size()); + std::vector statuses(keys.size()); + db_->MultiGet(read_options, db_->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + + for (size_t i = 0; i < keys.size(); ++i) { + ASSERT_OK(statuses[i]); + ASSERT_EQ(statuses[i].IsOkMergeOperandThresholdExceeded(), + threshold.has_value() && i + 1 > threshold.value()); + ASSERT_EQ(values[i], keys[i].ToString() + std::to_string(i + 1)); + } + } + }; + + // Test the case when the feature is disabled as well as various thresholds. + verify(std::nullopt); + for (size_t i = 0; i < 5; ++i) { + verify(i); + } + + // Flush and try again to test the case when results are served from SSTs. + ASSERT_OK(Flush()); + verify(std::nullopt); + for (size_t i = 0; i < 5; ++i) { + verify(i); + } +} + TEST_F(DBMergeOperatorTest, DataBlockBinaryAndHash) { // Basic test to check that merge operator works with data block index type // DataBlockBinaryAndHash. @@ -857,6 +951,98 @@ TEST_P(PerConfigMergeOperatorPinningTest, Randomized) { VerifyDBFromMap(true_data); } +TEST_F(DBMergeOperatorTest, MaxSuccessiveMergesBaseValues) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.merge_operator = MergeOperators::CreatePutOperator(); + options.max_successive_merges = 1; + options.env = env_; + Reopen(options); + + constexpr char foo[] = "foo"; + constexpr char bar[] = "bar"; + constexpr char baz[] = "baz"; + constexpr char qux[] = "qux"; + constexpr char corge[] = "corge"; + + // No base value + { + constexpr char key[] = "key1"; + + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, foo)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, bar)); + + PinnableSlice result; + ASSERT_OK( + db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result)); + ASSERT_EQ(result, bar); + + // We expect the second Merge to be converted to a Put because of + // max_successive_merges. + constexpr size_t max_key_versions = 8; + std::vector key_versions; + ASSERT_OK(GetAllKeyVersions(db_, db_->DefaultColumnFamily(), key, key, + max_key_versions, &key_versions)); + ASSERT_EQ(key_versions.size(), 2); + ASSERT_EQ(key_versions[0].type, kTypeValue); + ASSERT_EQ(key_versions[1].type, kTypeMerge); + } + + // Plain base value + { + constexpr char key[] = "key2"; + + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), key, foo)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, bar)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, baz)); + + PinnableSlice result; + ASSERT_OK( + db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result)); + ASSERT_EQ(result, baz); + + // We expect the second Merge to be converted to a Put because of + // max_successive_merges. + constexpr size_t max_key_versions = 8; + std::vector key_versions; + ASSERT_OK(GetAllKeyVersions(db_, db_->DefaultColumnFamily(), key, key, + max_key_versions, &key_versions)); + ASSERT_EQ(key_versions.size(), 3); + ASSERT_EQ(key_versions[0].type, kTypeValue); + ASSERT_EQ(key_versions[1].type, kTypeMerge); + ASSERT_EQ(key_versions[2].type, kTypeValue); + } + + // Wide-column base value + { + constexpr char key[] = "key3"; + const WideColumns columns{{kDefaultWideColumnName, foo}, {bar, baz}}; + + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, + columns)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, qux)); + ASSERT_OK( + db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, corge)); + + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), key, + &result)); + const WideColumns expected{{kDefaultWideColumnName, corge}, {bar, baz}}; + ASSERT_EQ(result.columns(), expected); + + // We expect the second Merge to be converted to a PutEntity because of + // max_successive_merges. + constexpr size_t max_key_versions = 8; + std::vector key_versions; + ASSERT_OK(GetAllKeyVersions(db_, db_->DefaultColumnFamily(), key, key, + max_key_versions, &key_versions)); + ASSERT_EQ(key_versions.size(), 3); + ASSERT_EQ(key_versions[0].type, kTypeWideColumnEntity); + ASSERT_EQ(key_versions[1].type, kTypeMerge); + ASSERT_EQ(key_versions[2].type, kTypeWideColumnEntity); + } +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/db_options_test.cc b/db/db_options_test.cc index d64d0eae5..8f60f0051 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -19,6 +19,8 @@ #include "rocksdb/convenience.h" #include "rocksdb/rate_limiter.h" #include "rocksdb/stats_history.h" +#include "rocksdb/utilities/options_util.h" +#include "test_util/mock_time_env.h" #include "test_util/sync_point.h" #include "test_util/testutil.h" #include "util/random.h" @@ -740,6 +742,55 @@ TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) { Close(); } +TEST_F(DBOptionsTest, SetStatsDumpPeriodSecRace) { + // This is a mini-stress test looking for inconsistency between the reported + // state of the option and the behavior in effect for the DB, after the last + // modification to that option (indefinite inconsistency). + std::vector threads; + for (int i = 0; i < 12; i++) { + threads.emplace_back([this, i]() { + ASSERT_OK(dbfull()->SetDBOptions( + {{"stats_dump_period_sec", i % 2 ? "100" : "0"}})); + }); + } + + for (auto& t : threads) { + t.join(); + } + + bool stats_dump_set = dbfull()->GetDBOptions().stats_dump_period_sec > 0; + bool task_enabled = dbfull()->TEST_GetPeriodicTaskScheduler().TEST_HasTask( + PeriodicTaskType::kDumpStats); + + ASSERT_EQ(stats_dump_set, task_enabled); +} + +TEST_F(DBOptionsTest, SetOptionsAndFileRace) { + // This is a mini-stress test looking for inconsistency between the reported + // state of the option and what is persisted in the options file, after the + // last modification to that option (indefinite inconsistency). + std::vector threads; + for (int i = 0; i < 12; i++) { + threads.emplace_back([this, i]() { + ASSERT_OK(dbfull()->SetOptions({{"ttl", std::to_string(i * 100)}})); + }); + } + + for (auto& t : threads) { + t.join(); + } + + auto setting_in_mem = dbfull()->GetOptions().ttl; + + std::vector cf_descs; + DBOptions db_options; + ConfigOptions cfg; + cfg.env = env_; + ASSERT_OK(LoadLatestOptions(cfg, dbname_, &db_options, &cf_descs, nullptr)); + ASSERT_EQ(cf_descs.size(), 1); + ASSERT_EQ(setting_in_mem, cf_descs[0].options.ttl); +} + TEST_F(DBOptionsTest, SetOptionsStatsPersistPeriodSec) { Options options; options.create_if_missing = true; @@ -908,6 +959,7 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { Options options; options.env = CurrentOptions().env; options.compaction_style = kCompactionStyleFIFO; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); options.write_buffer_size = 10 << 10; // 10KB options.arena_block_size = 4096; options.compression = kNoCompression; @@ -941,6 +993,9 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_EQ(NumTableFilesAtLevel(0), 10); + ASSERT_EQ(options.statistics->getTickerCount(FIFO_TTL_COMPACTIONS), 0); + ASSERT_EQ(options.statistics->getTickerCount(FIFO_MAX_SIZE_COMPACTIONS), 0); + // Set ttl to 1 minute. So all files should get deleted. ASSERT_OK(dbfull()->SetOptions({{"ttl", "60"}})); ASSERT_EQ(dbfull()->GetOptions().ttl, 60); @@ -948,6 +1003,10 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { ASSERT_OK(dbfull()->TEST_WaitForCompact()); ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_GT(options.statistics->getTickerCount(FIFO_TTL_COMPACTIONS), 0); + ASSERT_EQ(options.statistics->getTickerCount(FIFO_MAX_SIZE_COMPACTIONS), 0); + ASSERT_OK(options.statistics->Reset()); + // NOTE: Presumed unnecessary and removed: resetting mock time in env // Test dynamically changing compaction_options_fifo.max_table_files_size @@ -971,6 +1030,9 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_EQ(NumTableFilesAtLevel(0), 10); + ASSERT_EQ(options.statistics->getTickerCount(FIFO_MAX_SIZE_COMPACTIONS), 0); + ASSERT_EQ(options.statistics->getTickerCount(FIFO_TTL_COMPACTIONS), 0); + // Set max_table_files_size to 12 KB. So only 1 file should remain now. ASSERT_OK(dbfull()->SetOptions( {{"compaction_options_fifo", "{max_table_files_size=12288;}"}})); @@ -980,6 +1042,10 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { ASSERT_OK(dbfull()->TEST_WaitForCompact()); ASSERT_EQ(NumTableFilesAtLevel(0), 1); + ASSERT_GT(options.statistics->getTickerCount(FIFO_MAX_SIZE_COMPACTIONS), 0); + ASSERT_EQ(options.statistics->getTickerCount(FIFO_TTL_COMPACTIONS), 0); + ASSERT_OK(options.statistics->Reset()); + // Test dynamically changing compaction_options_fifo.allow_compaction options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB options.ttl = 0; @@ -1033,32 +1099,218 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { ASSERT_EQ(fifo_temp_opt[1].age, 30000); } -TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) { - SpecialEnv env(env_); +TEST_F(DBOptionsTest, OffpeakTimes) { Options options; - options.env = &env; + options.create_if_missing = true; + Random rnd(test::RandomSeed()); - options.compaction_readahead_size = 0; - options.level0_file_num_compaction_trigger = 2; - const std::string kValue(1024, 'v'); - Reopen(options); + auto verify_invalid = [&]() { + Status s = DBImpl::TEST_ValidateOptions(options); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + }; - ASSERT_EQ(0, dbfull()->GetDBOptions().compaction_readahead_size); - ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}})); - ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size); - for (int i = 0; i < 1024; i++) { - ASSERT_OK(Put(Key(i), kValue)); + auto verify_valid = [&]() { + Status s = DBImpl::TEST_ValidateOptions(options); + ASSERT_OK(s); + ASSERT_FALSE(s.IsInvalidArgument()); + }; + std::vector invalid_cases = { + "06:30-", + "-23:30", // Both need to be set + "00:00-00:00", + "06:30-06:30" // Start time cannot be the same as end time + "12:30 PM-23:30", + "12:01AM-11:00PM", // Invalid format + "01:99-22:00", // Invalid value for minutes + "00:00-24:00", // 24:00 is an invalid value + "6-7", + "6:-7", + "06:31.42-7:00", + "6.31:42-7:00", + "6:0-7:", + "15:0.2-3:.7", + ":00-00:02", + "02:00-:00", + "random-value", + "No:No-Hi:Hi", + }; + + std::vector valid_cases = { + "", // Not enabled. Valid case + "06:30-11:30", + "06:30-23:30", + "13:30-14:30", + "00:00-23:59", // Entire Day + "23:30-01:15", // From 11:30PM to 1:15AM next day. Valid case. + "1:0000000000000-2:000000000042", // Weird, but we can parse the int. + }; + + for (std::string invalid_case : invalid_cases) { + options.daily_offpeak_time_utc = invalid_case; + verify_invalid(); } - ASSERT_OK(Flush()); - for (int i = 0; i < 1024 * 2; i++) { - ASSERT_OK(Put(Key(i), kValue)); + for (std::string valid_case : valid_cases) { + options.daily_offpeak_time_utc = valid_case; + verify_valid(); } - ASSERT_OK(Flush()); - ASSERT_OK(dbfull()->TEST_WaitForCompact()); - ASSERT_EQ(256, env_->compaction_readahead_size_); + + auto verify_offpeak_info = [&](bool expected_is_now_off_peak, + int expected_seconds_till_next_offpeak_start, + int now_utc_hour, int now_utc_minute, + int now_utc_second = 0) { + auto mock_clock = std::make_shared(env_->GetSystemClock()); + // Add some extra random days to current time + int days = rnd.Uniform(100); + mock_clock->SetCurrentTime( + days * OffpeakTimeOption::kSecondsPerDay + + now_utc_hour * OffpeakTimeOption::kSecondsPerHour + + now_utc_minute * OffpeakTimeOption::kSecondsPerMinute + now_utc_second); + Status s = DBImpl::TEST_ValidateOptions(options); + ASSERT_OK(s); + auto offpeak_option = OffpeakTimeOption(options.daily_offpeak_time_utc); + int64_t now; + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + auto offpeak_info = offpeak_option.GetOffpeakTimeInfo(now); + ASSERT_EQ(expected_is_now_off_peak, offpeak_info.is_now_offpeak); + ASSERT_EQ(expected_seconds_till_next_offpeak_start, + offpeak_info.seconds_till_next_offpeak_start); + }; + + options.daily_offpeak_time_utc = ""; + verify_offpeak_info(false, 0, 12, 30); + + options.daily_offpeak_time_utc = "06:30-11:30"; + verify_offpeak_info(false, 1 * OffpeakTimeOption::kSecondsPerHour, 5, 30); + verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 6, 30); + verify_offpeak_info(true, 20 * OffpeakTimeOption::kSecondsPerHour, 10, 30); + verify_offpeak_info(true, 19 * OffpeakTimeOption::kSecondsPerHour, 11, 30); + verify_offpeak_info(false, 17 * OffpeakTimeOption::kSecondsPerHour, 13, 30); + + options.daily_offpeak_time_utc = "23:30-04:30"; + verify_offpeak_info(false, 17 * OffpeakTimeOption::kSecondsPerHour, 6, 30); + verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 23, 30); + verify_offpeak_info(true, + 23 * OffpeakTimeOption::kSecondsPerHour + + 30 * OffpeakTimeOption::kSecondsPerMinute, + 0, 0); + verify_offpeak_info(true, + 22 * OffpeakTimeOption::kSecondsPerHour + + 30 * OffpeakTimeOption::kSecondsPerMinute, + 1, 0); + verify_offpeak_info(true, 19 * OffpeakTimeOption::kSecondsPerHour, 4, 30); + verify_offpeak_info(false, + 18 * OffpeakTimeOption::kSecondsPerHour + + 59 * OffpeakTimeOption::kSecondsPerMinute, + 4, 31); + + // Entire day offpeak + options.daily_offpeak_time_utc = "00:00-23:59"; + verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 0, 0); + verify_offpeak_info(true, 12 * OffpeakTimeOption::kSecondsPerHour, 12, 00); + verify_offpeak_info(true, 1 * OffpeakTimeOption::kSecondsPerMinute, 23, 59); + verify_offpeak_info(true, 59, 23, 59, 1); + verify_offpeak_info(true, 1, 23, 59, 59); + + // Start with a valid option + options.daily_offpeak_time_utc = "01:30-04:15"; + DestroyAndReopen(options); + ASSERT_EQ("01:30-04:15", dbfull()->GetDBOptions().daily_offpeak_time_utc); + + int may_schedule_compaction_called = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::MaybeScheduleFlushOrCompaction:Start", + [&](void*) { may_schedule_compaction_called++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Make sure calling SetDBOptions with invalid option does not change the + // value nor call MaybeScheduleFlushOrCompaction() + for (std::string invalid_case : invalid_cases) { + ASSERT_NOK( + dbfull()->SetDBOptions({{"daily_offpeak_time_utc", invalid_case}})); + ASSERT_EQ("01:30-04:15", dbfull() + ->GetVersionSet() + ->offpeak_time_option() + .daily_offpeak_time_utc); + ASSERT_EQ(1 * kSecondInHour + 30 * kSecondInMinute, + dbfull() + ->GetVersionSet() + ->offpeak_time_option() + .daily_offpeak_start_time_utc); + ASSERT_EQ(4 * kSecondInHour + 15 * kSecondInMinute, + dbfull() + ->GetVersionSet() + ->offpeak_time_option() + .daily_offpeak_end_time_utc); + } + ASSERT_EQ(0, may_schedule_compaction_called); + + // Changing to new valid values should call MaybeScheduleFlushOrCompaction() + // and sets the offpeak_time_option in VersionSet + int expected_count = 0; + for (std::string valid_case : valid_cases) { + if (dbfull() + ->GetVersionSet() + ->offpeak_time_option() + .daily_offpeak_time_utc != valid_case) { + expected_count++; + } + ASSERT_OK(dbfull()->SetDBOptions({{"daily_offpeak_time_utc", valid_case}})); + ASSERT_EQ(valid_case, dbfull()->GetDBOptions().daily_offpeak_time_utc); + ASSERT_EQ(valid_case, dbfull() + ->GetVersionSet() + ->offpeak_time_option() + .daily_offpeak_time_utc); + } + ASSERT_EQ(expected_count, may_schedule_compaction_called); + + // Changing to the same value should not call MaybeScheduleFlushOrCompaction() + ASSERT_OK( + dbfull()->SetDBOptions({{"daily_offpeak_time_utc", "06:30-11:30"}})); + may_schedule_compaction_called = 0; + ASSERT_OK( + dbfull()->SetDBOptions({{"daily_offpeak_time_utc", "06:30-11:30"}})); + ASSERT_EQ(0, may_schedule_compaction_called); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); Close(); } +TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) { + for (bool use_direct_reads : {true, false}) { + SpecialEnv env(env_); + Options options; + options.env = &env; + + options.use_direct_reads = use_direct_reads; + options.level0_file_num_compaction_trigger = 2; + const std::string kValue(1024, 'v'); + Status s = TryReopen(options); + if (use_direct_reads && (s.IsNotSupported() || s.IsInvalidArgument())) { + continue; + } else { + ASSERT_OK(s); + } + + ASSERT_EQ(1024 * 1024 * 2, + dbfull()->GetDBOptions().compaction_readahead_size); + ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}})); + ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size); + for (int i = 0; i < 1024; i++) { + ASSERT_OK(Put(Key(i), kValue)); + } + ASSERT_OK(Flush()); + for (int i = 0; i < 1024 * 2; i++) { + ASSERT_OK(Put(Key(i), kValue)); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(256, env_->compaction_readahead_size_); + Close(); + } +} + TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) { Options options; options.compaction_style = kCompactionStyleFIFO; @@ -1307,7 +1559,7 @@ TEST_F(DBOptionsTest, TempOptionsFailTest) { [&](void* /*arg*/) { fs->SetFilesystemActive(true); }); SyncPoint::GetInstance()->EnableProcessing(); - TryReopen(options); + ASSERT_NOK(TryReopen(options)); SyncPoint::GetInstance()->DisableProcessing(); std::vector filenames; diff --git a/db/db_properties_test.cc b/db/db_properties_test.cc index d689686c3..e761f96d9 100644 --- a/db/db_properties_test.cc +++ b/db/db_properties_test.cc @@ -107,12 +107,12 @@ TEST_F(DBPropertiesTest, Empty) { dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num)); ASSERT_EQ("0", num); - ASSERT_OK(db_->EnableFileDeletions(false)); + ASSERT_OK(db_->EnableFileDeletions(/*force=*/false)); ASSERT_TRUE( dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num)); ASSERT_EQ("0", num); - ASSERT_OK(db_->EnableFileDeletions()); + ASSERT_OK(db_->EnableFileDeletions(/*force=*/true)); ASSERT_TRUE( dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num)); ASSERT_EQ("1", num); @@ -1084,12 +1084,14 @@ class CountingUserTblPropCollector : public TablePropertiesCollector { const char* Name() const override { return "CountingUserTblPropCollector"; } Status Finish(UserCollectedProperties* properties) override { + assert(!finish_called_); std::string encoded; PutVarint32(&encoded, count_); *properties = UserCollectedProperties{ {"CountingUserTblPropCollector", message_}, {"Count", encoded}, }; + finish_called_ = true; return Status::OK(); } @@ -1101,12 +1103,14 @@ class CountingUserTblPropCollector : public TablePropertiesCollector { } UserCollectedProperties GetReadableProperties() const override { + assert(finish_called_); return UserCollectedProperties{}; } private: std::string message_ = "Rocksdb"; uint32_t count_ = 0; + bool finish_called_ = false; }; class CountingUserTblPropCollectorFactory @@ -1740,7 +1744,7 @@ TEST_F(DBPropertiesTest, SstFilesSize) { ASSERT_EQ(obsolete_sst_size, sst_size); // Let the obsolete files be deleted. - ASSERT_OK(db_->EnableFileDeletions()); + ASSERT_OK(db_->EnableFileDeletions(/*force=*/false)); ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kObsoleteSstFilesSize, &obsolete_sst_size)); ASSERT_EQ(obsolete_sst_size, 0); @@ -2348,6 +2352,9 @@ TEST_F(DBPropertiesTest, TableMetaIndexKeys) { EXPECT_EQ("rocksdb.hashindex.prefixes", PopMetaIndexKey(meta_iter.get())); } + if (bbto->format_version >= 6) { + EXPECT_EQ("rocksdb.index", PopMetaIndexKey(meta_iter.get())); + } } EXPECT_EQ("rocksdb.properties", PopMetaIndexKey(meta_iter.get())); EXPECT_EQ("NOT_FOUND", PopMetaIndexKey(meta_iter.get())); diff --git a/db/db_range_del_test.cc b/db/db_range_del_test.cc index 5abb7dd2b..003117eec 100644 --- a/db/db_range_del_test.cc +++ b/db/db_range_del_test.cc @@ -682,6 +682,7 @@ TEST_F(DBRangeDelTest, TableEvictedDuringScan) { // soon as its refcount drops to zero. bbto.block_cache->EraseUnRefEntries(); } + ASSERT_OK(iter->status()); ASSERT_EQ(kNum, expected); delete iter; db_->ReleaseSnapshot(snapshot); @@ -840,6 +841,7 @@ TEST_F(DBRangeDelTest, IteratorRemovesCoveredKeys) { ++expected; } } + ASSERT_OK(iter->status()); ASSERT_EQ(kNum, expected); delete iter; } @@ -908,6 +910,7 @@ TEST_F(DBRangeDelTest, IteratorIgnoresRangeDeletions) { std::string key; ASSERT_EQ(expected[i], iter->key()); } + ASSERT_OK(iter->status()); ASSERT_EQ(3, i); delete iter; db_->ReleaseSnapshot(snapshot); @@ -1382,6 +1385,7 @@ TEST_F(DBRangeDelTest, UntruncatedTombstoneDoesNotDeleteNewerKey) { for (; iter->Valid(); iter->Next()) { ++keys_found; } + EXPECT_OK(iter->status()); delete iter; return keys_found; }; @@ -1485,6 +1489,7 @@ TEST_F(DBRangeDelTest, DeletedMergeOperandReappearsIterPrev) { for (; iter->Valid(); iter->Prev()) { ++keys_found; } + ASSERT_OK(iter->status()); delete iter; ASSERT_EQ(kNumKeys, keys_found); @@ -1519,6 +1524,7 @@ TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeys) { iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); delete iter; db_->ReleaseSnapshot(snapshot); @@ -1564,6 +1570,7 @@ TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeysInImmMemTables) { iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); } TEST_F(DBRangeDelTest, RangeTombstoneWrittenToMinimalSsts) { @@ -1978,6 +1985,7 @@ TEST_F(DBRangeDelTest, IteratorRefresh) { ASSERT_EQ("key1", iter->key()); iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); delete iter; } @@ -3475,6 +3483,292 @@ TEST_F(DBRangeDelTest, NonBottommostCompactionDropRangetombstone) { db_->ReleaseSnapshot(snapshot); } +TEST_F(DBRangeDelTest, MemtableMaxRangeDeletions) { + // Tests option `memtable_max_range_deletions`. + Options options = CurrentOptions(); + options.level_compaction_dynamic_file_size = false; + options.memtable_max_range_deletions = 50; + options.level0_file_num_compaction_trigger = 5; + DestroyAndReopen(options); + + for (int i = 0; i < 50; ++i) { + // Intentionally delete overlapping ranges to see if the option + // checks number of range tombstone fragments instead. + ASSERT_OK(Put(Key(i), "val1")); + ASSERT_OK(Put(Key(i + 1), "val2")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(i), Key(i + 2))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + } + // One more write to trigger flush. + ASSERT_OK(Put(Key(50), "val")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // This should take effect for the next new memtable. + ASSERT_OK(db_->SetOptions({{"memtable_max_range_deletions", "1"}})); + ASSERT_OK(Flush()); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(50), Key(100))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + // One more write to trigger flush. + ASSERT_OK(Put(Key(50), "new val")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(3, NumTableFilesAtLevel(0)); +} + +TEST_F(DBRangeDelTest, RangeDelReseekAfterFileReadError) { + // This is to test a bug that is fixed in + // https://github.com/facebook/rocksdb/pull/11786. + Options opts = CurrentOptions(); + opts.num_levels = 7; + + // Set up LSM + // + // L4: F1: [key1] F2: [key2] + // L5: F3:[DeleteRange(key3, key6)] + // L6: F4:[key3, key6] + // Will inject error when reading from F2. + // SeekToFirst() should land on key1. + // Next() should encounter error when reading from F2, + // and range del reseek should not reset this status. + Random rnd(301); + // L6 + ASSERT_OK(Put(Key(3), rnd.RandomString(100))); + ASSERT_OK(Put(Key(6), rnd.RandomString(100))); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + // L5 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(6))); + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + // L4 + ASSERT_OK(Put(Key(2), rnd.RandomString(100))); + ASSERT_OK(Flush()); + MoveFilesToLevel(4); + std::string fname; + std::vector live_files; + db_->GetLiveFilesMetaData(&live_files); + for (auto& meta : live_files) { + if (meta.level == 4) { + fname = meta.name; + break; + } + } + ASSERT_TRUE(!fname.empty()); + ASSERT_OK(Put(Key(1), rnd.RandomString(100))); + ASSERT_OK(Flush()); + MoveFilesToLevel(4); + + SyncPoint::GetInstance()->SetCallBack( + "RandomAccessFileReader::Read::BeforeReturn", [&fname](void* pair_ptr) { + auto p = + reinterpret_cast*>(pair_ptr); + if (p->first->find(fname) != std::string::npos) { + *p->second = IOStatus::IOError(); + p->second->SetRetryable(true); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + std::unique_ptr iter{db_->NewIterator(ReadOptions())}; + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), Key(1)); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + iter.reset(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + + // Reverse scan + // LSM setup + // L4: F1: [key2] F2: [key7, key8] + // L5: F3:[[key3, key6)] + // L6: F4:[key1, key5] + // Ingest error when read from F1. + // SeekToLast() should land on key8. + // During Prev(), MergingIterator will encounter error when reading from F1 + // and do a range del reseek (it sees key5 covered by a range tombstone). + DestroyAndReopen(opts); + // L6 + ASSERT_OK(Put(Key(1), rnd.RandomString(100))); + ASSERT_OK(Put(Key(5), rnd.RandomString(100))); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + // L5 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(6))); + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + // L4 + ASSERT_OK(Put(Key(2), rnd.RandomString(100))); + ASSERT_OK(Flush()); + MoveFilesToLevel(4); + live_files.clear(); + db_->GetLiveFilesMetaData(&live_files); + for (auto& meta : live_files) { + if (meta.level == 4) { + fname = meta.name; + break; + } + } + ASSERT_TRUE(!fname.empty()); + ASSERT_OK(Put(Key(7), rnd.RandomString(100))); + ASSERT_OK(Put(Key(8), rnd.RandomString(100))); + ASSERT_OK(Flush()); + MoveFilesToLevel(4); + + SyncPoint::GetInstance()->SetCallBack( + "RandomAccessFileReader::Read::AnyOffset", [&fname](void* pair_ptr) { + auto p = + reinterpret_cast*>(pair_ptr); + if (p->first->find(fname) != std::string::npos) { + *p->second = IOStatus::IOError(); + p->second->SetRetryable(true); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + iter.reset(db_->NewIterator(ReadOptions())); + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), Key(8)); + // Note that for reverse scan, DBIter will need to ensure + // the key it returns is the one with the highest sequence number. + // To return key7, it internally calls MergingIterator::Prev() + // until it reaches a previous user key. + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_NOK(iter->status()); + ASSERT_TRUE(iter->status().IsIOError()); + + iter.reset(); +} + +TEST_F(DBRangeDelTest, ReleaseSnapshotAfterIteratorCreation) { + // Test that range tombstone code path in LevelIterator + // does access ReadOptions::snapshot after Iterator creation. + // + // Put some data in L2 so that range tombstone in L1 will not be dropped. + ASSERT_OK(Put(Key(0), "v")); + ASSERT_OK(Put(Key(100), "v")); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + + // two L1 file with range del + ASSERT_OK(Put(Key(1), "v")); + ASSERT_OK(Put(Key(2), "v")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(4))); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + ASSERT_OK(Put(Key(5), "v")); + ASSERT_OK(Put(Key(6), "v")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(5), + Key(6))); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + const Snapshot* snapshot = db_->GetSnapshot(); + ReadOptions ro; + ro.snapshot = snapshot; + + Iterator* iter = db_->NewIterator(ro); + db_->ReleaseSnapshot(snapshot); + + iter->Seek(Key(1)); + std::vector expected_keys{1, 2, 6, 100}; + for (int i : expected_keys) { + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(i)); + iter->Next(); + } + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + + delete iter; +} + +TEST_F(DBRangeDelTest, RefreshWithSnapshot) { + ASSERT_OK(Put(Key(4), "4")); + ASSERT_OK(Put(Key(6), "6")); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(5))); + + std::unique_ptr iter{db_->NewIterator(ReadOptions())}; + // Live Memtable + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(6)); + ASSERT_OK(iter->Refresh(snapshot)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(4)); + // Immutable Memtable + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + ASSERT_OK(iter->Refresh(nullptr)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(6)); + ASSERT_OK(iter->Refresh(snapshot)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(4)); + // L0 + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_OK(iter->Refresh(nullptr)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(6)); + ASSERT_OK(iter->Refresh(snapshot)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(4)); + // L1 + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + ASSERT_OK(iter->Refresh(nullptr)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(6)); + ASSERT_OK(iter->Refresh(snapshot)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(4)); + // L1 with two file. + // Test that when LevelIterator enters a new file, + // it remembers which snapshot sequence number to use. + ASSERT_OK(Put(Key(2), "2")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + ASSERT_OK(iter->Refresh(nullptr)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + // LevelIterator is at the first file + ASSERT_EQ(iter->key(), Key(2)); + ASSERT_OK(iter->Refresh(snapshot)); + // Will enter the second file, and create a new range tombstone iterator. + // It should use the snapshot sequence number. + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(4)); + iter.reset(); + db_->ReleaseSnapshot(snapshot); +} } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/db_rate_limiter_test.cc b/db/db_rate_limiter_test.cc index 84c2df230..05419db44 100644 --- a/db/db_rate_limiter_test.cc +++ b/db/db_rate_limiter_test.cc @@ -220,6 +220,7 @@ TEST_P(DBRateLimiterOnReadTest, Iterator) { ++expected; } } + ASSERT_OK(iter->status()); // Reverse scan does not read evenly (one block per iteration) due to // descending seqno ordering, so wait until after the loop to check total. ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); @@ -235,9 +236,18 @@ TEST_P(DBRateLimiterOnReadTest, VerifyChecksum) { ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); ASSERT_OK(db_->VerifyChecksum(GetReadOptions())); - // There are 3 reads per file: ReadMetaIndexBlock, - // VerifyChecksumInMetaBlocks, VerifyChecksumInBlocks - int expected = kNumFiles * 3; + // In BufferedIO, + // there are 7 reads per file, each of which will be rate-limited. + // During open: read footer, meta index block, properties block, index block. + // During actual checksum verification: read meta index block, verify checksum + // in meta blocks and verify checksum in file blocks. + // + // In DirectIO, where we support tail prefetching, during table open, we only + // do 1 read instead of 4 as described above. Actual checksum verification + // reads stay the same. + int num_read_per_file = (!use_direct_io_) ? 7 : 4; + int expected = kNumFiles * num_read_per_file; + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); } diff --git a/db/db_readonly_with_timestamp_test.cc b/db/db_readonly_with_timestamp_test.cc index 675e4943b..7a37bfec8 100644 --- a/db/db_readonly_with_timestamp_test.cc +++ b/db/db_readonly_with_timestamp_test.cc @@ -240,6 +240,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) { get_value_and_check(db_, read_opts, it->key(), it->value(), write_timestamps[i]); } + ASSERT_OK(it->status()); size_t expected_count = kMaxKey - start_keys[i] + 1; ASSERT_EQ(expected_count, count); @@ -252,6 +253,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) { get_value_and_check(db_, read_opts, it->key(), it->value(), write_timestamps[i]); } + ASSERT_OK(it->status()); ASSERT_EQ(static_cast(kMaxKey) - start_keys[i] + 1, count); // SeekToFirst()/SeekToLast() with lower/upper bounds. @@ -273,6 +275,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) { get_value_and_check(db_, read_opts, it->key(), it->value(), write_timestamps[i]); } + ASSERT_OK(it->status()); ASSERT_EQ(r - std::max(l, start_keys[i]), count); for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0; @@ -282,6 +285,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) { get_value_and_check(db_, read_opts, it->key(), it->value(), write_timestamps[i]); } + ASSERT_OK(it->status()); l += (kMaxKey / 100); r -= (kMaxKey / 100); } @@ -328,6 +332,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, Iterators) { CheckIterUserEntry(iters[0], Key1(key), kTypeValue, "value" + std::to_string(key), write_timestamp); } + ASSERT_OK(iters[0]->status()); size_t expected_count = kMaxKey - 0 + 1; ASSERT_EQ(expected_count, count); @@ -336,6 +341,53 @@ TEST_F(DBReadOnlyTestWithTimestamp, Iterators) { Close(); } +TEST_F(DBReadOnlyTestWithTimestamp, FullHistoryTsLowSanityCheckFail) { + Options options = CurrentOptions(); + options.env = env_; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + // Use UDT in memtable only feature for this test, so we can control that + // newly set `full_history_ts_low` collapse history when Flush happens. + options.persist_user_defined_timestamps = false; + options.allow_concurrent_memtable_write = false; + DestroyAndReopen(options); + + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val1")); + + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 3); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + full_history_ts_low)); + ASSERT_OK(Flush(0)); + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + + // Reading below full_history_ts_low fails a sanity check. + std::string read_ts; + PutFixed64(&read_ts, 2); + Slice read_ts_slice = read_ts; + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + + // Get() + std::string value; + ASSERT_TRUE(db_->Get(read_opts, "foo", &value).IsInvalidArgument()); + // NewIterator() + std::unique_ptr iter( + db_->NewIterator(read_opts, db_->DefaultColumnFamily())); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + + // NewIterators() + std::vector cfhs = {db_->DefaultColumnFamily()}; + std::vector iterators; + ASSERT_TRUE( + db_->NewIterators(read_opts, cfhs, &iterators).IsInvalidArgument()); + Close(); +} + TEST_F(DBReadOnlyTestWithTimestamp, IteratorsReadTimestampSizeMismatch) { const int kNumKeysPerFile = 128; const uint64_t kMaxKey = 1024; diff --git a/db/db_secondary_test.cc b/db/db_secondary_test.cc index f3f0a8d05..8353790c3 100644 --- a/db/db_secondary_test.cc +++ b/db/db_secondary_test.cc @@ -130,7 +130,7 @@ TEST_F(DBSecondaryTest, FailOpenIfLoggerCreationFail) { SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->SetCallBack( - "rocksdb::CreateLoggerFromOptions:AfterGetPath", [&](void* arg) { + "forstdb::CreateLoggerFromOptions:AfterGetPath", [&](void* arg) { auto* s = reinterpret_cast(arg); assert(s); *s = Status::IOError("Injected"); @@ -164,12 +164,22 @@ TEST_F(DBSecondaryTest, ReopenAsSecondary) { Reopen(options); ASSERT_OK(Put("foo", "foo_value")); ASSERT_OK(Put("bar", "bar_value")); + WideColumns columns{{kDefaultWideColumnName, "attr_default_val"}, + {"attr_name1", "attr_value_1"}, + {"attr_name2", "attr_value_2"}}; + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), "baz", + columns)); ASSERT_OK(dbfull()->Flush(FlushOptions())); Close(); ASSERT_OK(ReopenAsSecondary(options)); ASSERT_EQ("foo_value", Get("foo")); ASSERT_EQ("bar_value", Get("bar")); + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), "baz", + &result)); + ASSERT_EQ(result.columns(), columns); + ReadOptions ropts; ropts.verify_checksums = true; auto db1 = static_cast(db_); @@ -182,13 +192,17 @@ TEST_F(DBSecondaryTest, ReopenAsSecondary) { ASSERT_EQ("bar", iter->key().ToString()); ASSERT_EQ("bar_value", iter->value().ToString()); } else if (1 == count) { + ASSERT_EQ("baz", iter->key().ToString()); + ASSERT_EQ(columns, iter->columns()); + } else if (2 == count) { ASSERT_EQ("foo", iter->key().ToString()); ASSERT_EQ("foo_value", iter->value().ToString()); } ++count; } + ASSERT_OK(iter->status()); delete iter; - ASSERT_EQ(2, count); + ASSERT_EQ(3, count); } TEST_F(DBSecondaryTest, SimpleInternalCompaction) { @@ -521,6 +535,8 @@ TEST_F(DBSecondaryTest, SecondaryCloseFiles) { } ASSERT_FALSE(iter1->Valid()); ASSERT_FALSE(iter2->Valid()); + ASSERT_OK(iter1->status()); + ASSERT_OK(iter2->status()); }; ASSERT_OK(Put("a", "value")); @@ -793,6 +809,7 @@ TEST_F(DBSecondaryTest, MissingTableFileDuringOpen) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ++count; } + ASSERT_OK(iter->status()); ASSERT_EQ(2, count); delete iter; } @@ -850,6 +867,7 @@ TEST_F(DBSecondaryTest, MissingTableFile) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ++count; } + ASSERT_OK(iter->status()); ASSERT_EQ(2, count); delete iter; } @@ -922,6 +940,7 @@ TEST_F(DBSecondaryTest, SwitchManifest) { ASSERT_EQ("value_" + std::to_string(kNumFiles - 1), iter->value().ToString()); } + EXPECT_OK(iter->status()); }; range_scan_db(); @@ -1472,6 +1491,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) { get_value_and_check(db_, read_opts, it->key(), it->value(), write_timestamps[i]); } + ASSERT_OK(it->status()); size_t expected_count = kMaxKey - start_keys[i] + 1; ASSERT_EQ(expected_count, count); @@ -1484,6 +1504,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) { get_value_and_check(db_, read_opts, it->key(), it->value(), write_timestamps[i]); } + ASSERT_OK(it->status()); ASSERT_EQ(static_cast(kMaxKey) - start_keys[i] + 1, count); // SeekToFirst()/SeekToLast() with lower/upper bounds. @@ -1505,6 +1526,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) { get_value_and_check(db_, read_opts, it->key(), it->value(), write_timestamps[i]); } + ASSERT_OK(it->status()); ASSERT_EQ(r - std::max(l, start_keys[i]), count); for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0; @@ -1514,6 +1536,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) { get_value_and_check(db_, read_opts, it->key(), it->value(), write_timestamps[i]); } + ASSERT_OK(it->status()); l += (kMaxKey / 100); r -= (kMaxKey / 100); } @@ -1561,6 +1584,55 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorsReadTimestampSizeMismatch) { Close(); } +TEST_F(DBSecondaryTestWithTimestamp, FullHistoryTsLowSanityCheckFail) { + Options options = CurrentOptions(); + options.env = env_; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + // Use UDT in memtable only feature for this test, so we can control that + // newly set `full_history_ts_low` collapse history when Flush happens. + options.persist_user_defined_timestamps = false; + options.allow_concurrent_memtable_write = false; + DestroyAndReopen(options); + + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val1")); + + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 3); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + full_history_ts_low)); + ASSERT_OK(Flush(0)); + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + // Reading below full_history_ts_low fails a sanity check. + std::string read_ts; + PutFixed64(&read_ts, 2); + Slice read_ts_slice = read_ts; + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + + // Get() + std::string value; + ASSERT_TRUE(db_->Get(read_opts, "foo", &value).IsInvalidArgument()); + + // NewIterator() + std::unique_ptr iter( + db_->NewIterator(read_opts, db_->DefaultColumnFamily())); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + + // NewIterators() + std::vector cfhs = {db_->DefaultColumnFamily()}; + std::vector iterators; + ASSERT_TRUE( + db_->NewIterators(read_opts, cfhs, &iterators).IsInvalidArgument()); + Close(); +} + TEST_F(DBSecondaryTestWithTimestamp, IteratorsReadTimestampSpecifiedWithoutWriteTimestamp) { const int kNumKeysPerFile = 128; @@ -1674,6 +1746,7 @@ TEST_F(DBSecondaryTestWithTimestamp, Iterators) { CheckIterUserEntry(iters[0], Key1(key), kTypeValue, "value" + std::to_string(key), write_timestamp); } + ASSERT_OK(iters[0]->status()); size_t expected_count = kMaxKey - 0 + 1; ASSERT_EQ(expected_count, count); diff --git a/db/db_sst_test.cc b/db/db_sst_test.cc index 4293e77a1..7590aa2f1 100644 --- a/db/db_sst_test.cc +++ b/db/db_sst_test.cc @@ -937,12 +937,21 @@ INSTANTIATE_TEST_CASE_P(DBWALTestWithParam, DBWALTestWithParam, ::testing::Values(std::make_tuple("", true), std::make_tuple("_wal_dir", false))); -TEST_F(DBSSTTest, OpenDBWithExistingTrash) { +TEST_F(DBSSTTest, OpenDBWithExistingTrashAndObsoleteSstFile) { Options options = CurrentOptions(); - options.sst_file_manager.reset( NewSstFileManager(env_, nullptr, "", 1024 * 1024 /* 1 MB/sec */)); auto sfm = static_cast(options.sst_file_manager.get()); + // Set an extra high trash ratio to prevent immediate/non-rate limited + // deletions + sfm->SetDeleteRateBytesPerSecond(1024 * 1024); + sfm->delete_scheduler()->SetMaxTrashDBRatio(1000.0); + + int bg_delete_file = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void* /*arg*/) { bg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); Destroy(last_options_); @@ -951,13 +960,22 @@ TEST_F(DBSSTTest, OpenDBWithExistingTrash) { ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "001.sst.trash")); ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "002.sst.trash")); ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "003.sst.trash")); - - // Reopen the DB and verify that it deletes existing trash files + // Manually add an obsolete sst file. Obsolete SST files are discovered and + // deleted upon recovery. + constexpr uint64_t kSstFileNumber = 100; + const std::string kObsoleteSstFile = + MakeTableFileName(dbname_, kSstFileNumber); + ASSERT_OK(WriteStringToFile(env_, "abc", kObsoleteSstFile)); + + // Reopen the DB and verify that it deletes existing trash files and obsolete + // SST files with rate limiting. Reopen(options); sfm->WaitForEmptyTrash(); ASSERT_NOK(env_->FileExists(dbname_ + "/" + "001.sst.trash")); ASSERT_NOK(env_->FileExists(dbname_ + "/" + "002.sst.trash")); ASSERT_NOK(env_->FileExists(dbname_ + "/" + "003.sst.trash")); + ASSERT_NOK(env_->FileExists(kObsoleteSstFile)); + ASSERT_EQ(bg_delete_file, 4); } // Create a DB with 2 db_paths, and generate multiple files in the 2 @@ -1520,6 +1538,11 @@ TEST_F(DBSSTTest, OpenDBWithInfiniteMaxOpenFilesSubjectToMemoryLimit) { } TEST_F(DBSSTTest, GetTotalSstFilesSize) { + // FIXME: L0 file and L1+ file also differ in size of `oldest_key_time`. + // L0 file has non-zero `oldest_key_time` while L1+ files have 0. + // The test passes since L1+ file uses current time instead of 0 + // as oldest_ancestor_time. + // // We don't propagate oldest-key-time table property on compaction and // just write 0 as default value. This affect the exact table size, since // we encode table properties as varint64. Force time to be 0 to work around diff --git a/db/db_statistics_test.cc b/db/db_statistics_test.cc index 8b291acda..f430811d3 100644 --- a/db/db_statistics_test.cc +++ b/db/db_statistics_test.cc @@ -6,9 +6,11 @@ #include #include "db/db_test_util.h" +#include "db/write_batch_internal.h" #include "monitoring/thread_status_util.h" #include "port/stack_trace.h" #include "rocksdb/statistics.h" +#include "rocksdb/utilities/transaction_db.h" #include "util/random.h" namespace ROCKSDB_NAMESPACE { @@ -219,7 +221,7 @@ TEST_F(DBStatisticsTest, VerifyChecksumReadStat) { ASSERT_OK(Flush()); std::unordered_map table_files; uint64_t table_files_size = 0; - GetAllDataFiles(kTableFile, &table_files, &table_files_size); + ASSERT_OK(GetAllDataFiles(kTableFile, &table_files, &table_files_size)); { // Scenario 1: Table verified in `VerifyFileChecksums()`. This should read @@ -283,6 +285,78 @@ TEST_F(DBStatisticsTest, BlockChecksumStats) { options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT)); } +TEST_F(DBStatisticsTest, BytesWrittenStats) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kExceptHistogramOrTimers); + Reopen(options); + + EXPECT_EQ(0, options.statistics->getAndResetTickerCount(WAL_FILE_BYTES)); + EXPECT_EQ(0, options.statistics->getAndResetTickerCount(BYTES_WRITTEN)); + + const int kNumKeysWritten = 100; + + // Scenario 0: Not using transactions. + // This will write to WAL and memtable directly. + ASSERT_OK(options.statistics->Reset()); + + for (int i = 0; i < kNumKeysWritten; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + + EXPECT_EQ(options.statistics->getAndResetTickerCount(WAL_FILE_BYTES), + options.statistics->getAndResetTickerCount(BYTES_WRITTEN)); + + // Scenario 1: Using transactions. + // This should not double count BYTES_WRITTEN (issue #12061). + for (bool enable_pipelined_write : {false, true}) { + ASSERT_OK(options.statistics->Reset()); + + // Destroy the DB to recreate as a TransactionDB. + Destroy(options, true); + + // Create a TransactionDB. + TransactionDB* txn_db = nullptr; + TransactionDBOptions txn_db_opts; + txn_db_opts.write_policy = TxnDBWritePolicy::WRITE_COMMITTED; + options.enable_pipelined_write = enable_pipelined_write; + ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, &txn_db)); + ASSERT_NE(txn_db, nullptr); + db_ = txn_db->GetBaseDB(); + + WriteOptions wopts; + TransactionOptions txn_opts; + Transaction* txn = txn_db->BeginTransaction(wopts, txn_opts, nullptr); + ASSERT_NE(txn, nullptr); + ASSERT_OK(txn->SetName("txn1")); + + for (int i = 0; i < kNumKeysWritten; ++i) { + ASSERT_OK(txn->Put(Key(i), "val")); + } + + // Prepare() writes to WAL, but not to memtable. (WriteCommitted) + ASSERT_OK(txn->Prepare()); + EXPECT_NE(0, options.statistics->getTickerCount(WAL_FILE_BYTES)); + // BYTES_WRITTEN would have been non-zero previously (issue #12061). + EXPECT_EQ(0, options.statistics->getTickerCount(BYTES_WRITTEN)); + + // Commit() writes to memtable and also a commit marker to WAL. + ASSERT_OK(txn->Commit()); + delete txn; + + // The WAL has an extra header of size `kHeader` written to it, + // as we are writing twice to it (first during Prepare, second during + // Commit). + EXPECT_EQ(options.statistics->getAndResetTickerCount(WAL_FILE_BYTES), + options.statistics->getAndResetTickerCount(BYTES_WRITTEN) + + WriteBatchInternal::kHeader); + + // Cleanup + db_ = nullptr; + delete txn_db; + } +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/db_table_properties_test.cc b/db/db_table_properties_test.cc index 61dcf3c1e..bfa67226e 100644 --- a/db/db_table_properties_test.cc +++ b/db/db_table_properties_test.cc @@ -22,6 +22,7 @@ #include "table/table_properties_internal.h" #include "test_util/testharness.h" #include "test_util/testutil.h" +#include "util/atomic.h" #include "util/random.h" @@ -417,6 +418,71 @@ TEST_F(DBTablePropertiesTest, GetDbIdentifiersProperty) { } } +TEST_F(DBTablePropertiesTest, FactoryReturnsNull) { + struct JunkTablePropertiesCollector : public TablePropertiesCollector { + const char* Name() const override { return "JunkTablePropertiesCollector"; } + Status Finish(UserCollectedProperties* properties) override { + properties->insert({"Junk", "Junk"}); + return Status::OK(); + } + UserCollectedProperties GetReadableProperties() const override { + return {}; + } + }; + + // Alternates between putting a "Junk" property and using `nullptr` to + // opt out. + static RelaxedAtomic count{0}; + struct SometimesTablePropertiesCollectorFactory + : public TablePropertiesCollectorFactory { + const char* Name() const override { + return "SometimesTablePropertiesCollectorFactory"; + } + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + if (count.FetchAddRelaxed(1) & 1) { + return nullptr; + } else { + return new JunkTablePropertiesCollector(); + } + } + }; + + Options options = CurrentOptions(); + options.table_properties_collector_factories.emplace_back( + std::make_shared()); + // For plain table + options.prefix_extractor.reset(NewFixedPrefixTransform(4)); + for (std::shared_ptr tf : + {options.table_factory, + std::shared_ptr(NewPlainTableFactory({}))}) { + SCOPED_TRACE("Table factory = " + std::string(tf->Name())); + options.table_factory = tf; + + DestroyAndReopen(options); + + ASSERT_OK(Put("key0", "value1")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key0", "value2")); + ASSERT_OK(Flush()); + + TablePropertiesCollection props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&props)); + int no_junk_count = 0; + int junk_count = 0; + for (const auto& item : props) { + if (item.second->user_collected_properties.find("Junk") != + item.second->user_collected_properties.end()) { + junk_count++; + } else { + no_junk_count++; + } + } + EXPECT_EQ(1, no_junk_count); + EXPECT_EQ(1, junk_count); + } +} + class DBTableHostnamePropertyTest : public DBTestBase, public ::testing::WithParamInterface> { diff --git a/db/db_tailing_iter_test.cc b/db/db_tailing_iter_test.cc index d3debed7e..07ffadc2a 100644 --- a/db/db_tailing_iter_test.cc +++ b/db/db_tailing_iter_test.cc @@ -52,6 +52,7 @@ TEST_P(DBTestTailingIterator, TailingIteratorSingle) { iter->Next(); ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); } TEST_P(DBTestTailingIterator, TailingIteratorKeepAdding) { @@ -361,7 +362,7 @@ TEST_P(DBTestTailingIterator, TailingIteratorDeletes) { int count = 0; for (; iter->Valid(); iter->Next(), ++count) ; - + ASSERT_OK(iter->status()); ASSERT_EQ(count, num_records); } Close(); @@ -408,6 +409,7 @@ TEST_P(DBTestTailingIterator, TailingIteratorPrefixSeek) { iter->Next(); ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); } Close(); } diff --git a/db/db_test.cc b/db/db_test.cc index 609f96ea5..99a03b150 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -127,6 +127,7 @@ TEST_F(DBTest, MockEnvTest) { iterator->Next(); } ASSERT_TRUE(!iterator->Valid()); + ASSERT_OK(iterator->status()); delete iterator; DBImpl* dbi = static_cast_with_check(db); @@ -171,6 +172,7 @@ TEST_F(DBTest, MemEnvTest) { iterator->Next(); } ASSERT_TRUE(!iterator->Valid()); + ASSERT_OK(iterator->status()); delete iterator; DBImpl* dbi = static_cast_with_check(db); @@ -644,6 +646,33 @@ TEST_F(DBTest, ReadFromPersistedTier) { ASSERT_OK(db_->Get(ropt, handles_[1], "bar", &value)); } + const auto check_multiget_func = + [&](const ReadOptions& read_opts, + std::vector cfhs, std::vector& keys, + std::vector& values, + bool batched) -> std::vector { + if (!batched) { + return db_->MultiGet(read_opts, cfhs, keys, &values); + } else { + size_t num_keys = keys.size(); + std::vector statuses; + std::vector pinnable_values; + statuses.resize(num_keys); + pinnable_values.resize(num_keys); + values.resize(num_keys); + db_->MultiGet(read_opts, cfhs[0], num_keys, keys.data(), + pinnable_values.data(), statuses.data(), false); + for (size_t i = 0; i < statuses.size(); ++i) { + if (statuses[i].ok()) { + values[i].assign(pinnable_values[i].data(), + pinnable_values[i].size()); + pinnable_values[i].Reset(); + } + } + return statuses; + } + }; + // Multiget std::vector multiget_cfs; multiget_cfs.push_back(handles_[1]); @@ -652,14 +681,17 @@ TEST_F(DBTest, ReadFromPersistedTier) { multiget_keys.push_back("foo"); multiget_keys.push_back("bar"); std::vector multiget_values; - auto statuses = - db_->MultiGet(ropt, multiget_cfs, multiget_keys, &multiget_values); - if (wopt.disableWAL) { - ASSERT_TRUE(statuses[0].IsNotFound()); - ASSERT_TRUE(statuses[1].IsNotFound()); - } else { - ASSERT_OK(statuses[0]); - ASSERT_OK(statuses[1]); + for (int i = 0; i < 2; i++) { + bool batched = i == 0; + auto statuses = check_multiget_func(ropt, multiget_cfs, multiget_keys, + multiget_values, batched); + if (wopt.disableWAL) { + ASSERT_TRUE(statuses[0].IsNotFound()); + ASSERT_TRUE(statuses[1].IsNotFound()); + } else { + ASSERT_OK(statuses[0]); + ASSERT_OK(statuses[1]); + } } // 2nd round: flush and put a new value in memtable. @@ -683,21 +715,26 @@ TEST_F(DBTest, ReadFromPersistedTier) { // Expect same result in multiget multiget_cfs.push_back(handles_[1]); multiget_keys.push_back("rocksdb"); - statuses = - db_->MultiGet(ropt, multiget_cfs, multiget_keys, &multiget_values); - ASSERT_TRUE(statuses[0].ok()); - ASSERT_EQ("first", multiget_values[0]); - ASSERT_TRUE(statuses[1].ok()); - ASSERT_EQ("one", multiget_values[1]); - if (wopt.disableWAL) { - ASSERT_TRUE(statuses[2].IsNotFound()); - } else { - ASSERT_OK(statuses[2]); + multiget_values.clear(); + + for (int i = 0; i < 2; i++) { + bool batched = i == 0; + auto statuses = check_multiget_func(ropt, multiget_cfs, multiget_keys, + multiget_values, batched); + ASSERT_TRUE(statuses[0].ok()); + ASSERT_EQ("first", multiget_values[0]); + ASSERT_TRUE(statuses[1].ok()); + ASSERT_EQ("one", multiget_values[1]); + if (wopt.disableWAL) { + ASSERT_TRUE(statuses[2].IsNotFound()); + } else { + ASSERT_OK(statuses[2]); + } } // 3rd round: delete and flush ASSERT_OK(db_->Delete(wopt, handles_[1], "foo")); - Flush(1); + ASSERT_OK(Flush(1)); ASSERT_OK(db_->Delete(wopt, handles_[1], "bar")); ASSERT_TRUE(db_->Get(ropt, handles_[1], "foo", &value).IsNotFound()); @@ -712,17 +749,21 @@ TEST_F(DBTest, ReadFromPersistedTier) { ASSERT_TRUE(db_->Get(ropt, handles_[1], "rocksdb", &value).ok()); ASSERT_EQ(value, "hello"); - statuses = - db_->MultiGet(ropt, multiget_cfs, multiget_keys, &multiget_values); - ASSERT_TRUE(statuses[0].IsNotFound()); - if (wopt.disableWAL) { - ASSERT_TRUE(statuses[1].ok()); - ASSERT_EQ("one", multiget_values[1]); - } else { - ASSERT_TRUE(statuses[1].IsNotFound()); + multiget_values.clear(); + for (int i = 0; i < 2; i++) { + bool batched = i == 0; + auto statuses = check_multiget_func(ropt, multiget_cfs, multiget_keys, + multiget_values, batched); + ASSERT_TRUE(statuses[0].IsNotFound()); + if (wopt.disableWAL) { + ASSERT_TRUE(statuses[1].ok()); + ASSERT_EQ("one", multiget_values[1]); + } else { + ASSERT_TRUE(statuses[1].IsNotFound()); + } + ASSERT_TRUE(statuses[2].ok()); + ASSERT_EQ("hello", multiget_values[2]); } - ASSERT_TRUE(statuses[2].ok()); - ASSERT_EQ("hello", multiget_values[2]); if (wopt.disableWAL == 0) { DestroyAndReopen(options); } @@ -860,7 +901,7 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) { ASSERT_EQ('w', value[0]); // Compact all files. - Flush(); + ASSERT_OK(Flush()); db_->CompactRange(CompactRangeOptions(), nullptr, nullptr); // Check DB is not in read-only state. @@ -1300,7 +1341,7 @@ TEST_F(DBTest, MetaDataTest) { options.disable_auto_compactions = true; int64_t temp_time = 0; - options.env->GetCurrentTime(&temp_time); + ASSERT_OK(options.env->GetCurrentTime(&temp_time)); uint64_t start_time = static_cast(temp_time); DestroyAndReopen(options); @@ -1329,7 +1370,7 @@ TEST_F(DBTest, MetaDataTest) { std::vector> files_by_level; dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &files_by_level); - options.env->GetCurrentTime(&temp_time); + ASSERT_OK(options.env->GetCurrentTime(&temp_time)); uint64_t end_time = static_cast(temp_time); ColumnFamilyMetaData cf_meta; @@ -2944,6 +2985,7 @@ TEST_F(DBTest, GroupCommitTest) { itr->Next(); } ASSERT_TRUE(!itr->Valid()); + ASSERT_OK(itr->status()); delete itr; HistogramData hist_data; @@ -3472,6 +3514,8 @@ static bool CompareIterators(int step, DB* model, DB* db, ok = false; } } + EXPECT_OK(miter->status()); + EXPECT_OK(dbiter->status()); (void)count; delete miter; delete dbiter; @@ -3648,7 +3692,7 @@ TEST_F(DBTest, BlockBasedTablePrefixHashIndexTest) { ASSERT_OK(Put("kk2", "v2")); ASSERT_OK(Put("kk", "v3")); ASSERT_OK(Put("k", "v4")); - Flush(); + ASSERT_OK(Flush()); ASSERT_EQ("v1", Get("kk1")); ASSERT_EQ("v2", Get("kk2")); @@ -4280,8 +4324,8 @@ TEST_F(DBTest, ConcurrentMemtableNotSupported) { options.soft_pending_compaction_bytes_limit = 0; options.hard_pending_compaction_bytes_limit = 100; options.create_if_missing = true; - - DestroyDB(dbname_, options); + Close(); + ASSERT_OK(DestroyDB(dbname_, options)); options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true, 4)); ASSERT_NOK(TryReopen(options)); @@ -4622,7 +4666,7 @@ TEST_F(DBTest, GetThreadStatus) { Options options; options.env = env_; options.enable_thread_tracking = true; - TryReopen(options); + ASSERT_OK(TryReopen(options)); std::vector thread_list; Status s = env_->GetThreadList(&thread_list); @@ -4693,7 +4737,7 @@ TEST_F(DBTest, DisableThreadStatus) { Options options; options.env = env_; options.enable_thread_tracking = false; - TryReopen(options); + ASSERT_OK(TryReopen(options)); CreateAndReopenWithCF({"pikachu", "about-to-remove"}, options); // Verify non of the column family info exists env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(handles_, @@ -4902,7 +4946,7 @@ TEST_P(DBTestWithParam, PreShutdownMultipleCompaction) { options.level0_slowdown_writes_trigger = 1 << 10; options.max_subcompactions = max_subcompactions_; - TryReopen(options); + ASSERT_OK(TryReopen(options)); Random rnd(301); std::vector thread_list; @@ -4991,7 +5035,7 @@ TEST_P(DBTestWithParam, PreShutdownCompactionMiddle) { options.level0_slowdown_writes_trigger = 1 << 10; options.max_subcompactions = max_subcompactions_; - TryReopen(options); + ASSERT_OK(TryReopen(options)); Random rnd(301); std::vector thread_list; @@ -5969,6 +6013,7 @@ TEST_F(DBTest, MergeTestTime) { ASSERT_OK(iter->status()); ++count; } + ASSERT_OK(iter->status()); ASSERT_EQ(1, count); ASSERT_EQ(4000000, TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME)); @@ -6040,6 +6085,64 @@ TEST_P(DBTestWithParam, FilterCompactionTimeTest) { delete itr; } +#ifndef OS_WIN +// CPUMicros() is not supported. See WinClock::CPUMicros(). +TEST_P(DBTestWithParam, CompactionTotalTimeTest) { + int record_count = 0; + class TestStatistics : public StatisticsImpl { + public: + explicit TestStatistics(int* record_count) + : StatisticsImpl(nullptr), record_count_(record_count) {} + void recordTick(uint32_t ticker_type, uint64_t count) override { + if (ticker_type == COMPACTION_CPU_TOTAL_TIME) { + ASSERT_GT(count, 0); + (*record_count_)++; + } + StatisticsImpl::recordTick(ticker_type, count); + } + + int* record_count_; + }; + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.create_if_missing = true; + options.statistics = std::make_shared(&record_count); + options.statistics->set_stats_level(kExceptTimeForMutex); + options.max_subcompactions = max_subcompactions_; + DestroyAndReopen(options); + + int n = 0; + for (int table = 0; table < 4; ++table) { + for (int i = 0; i < 1000; ++i) { + ASSERT_OK(Put(std::to_string(table * 1000 + i), "val")); + ++n; + } + // Overlapping tables + ASSERT_OK(Put(std::to_string(0), "val")); + ++n; + ASSERT_OK(Flush()); + } + + CompactRangeOptions cro; + cro.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // Hard-coded number in CompactionJob::ProcessKeyValueCompaction(). + const int kRecordStatsEvery = 1000; + // The stat COMPACTION_CPU_TOTAL_TIME should be recorded + // during compaction and once more after compaction. + ASSERT_EQ(n / kRecordStatsEvery + 1, record_count); + + // Check that COMPACTION_CPU_TOTAL_TIME correctly + // records compaction time after a compaction. + HistogramData h; + options.statistics->histogramData(COMPACTION_CPU_TIME, &h); + ASSERT_EQ(1, h.count); + ASSERT_EQ(h.max, TestGetTickerCount(options, COMPACTION_CPU_TOTAL_TIME)); +} +#endif + TEST_F(DBTest, TestLogCleanup) { Options options = CurrentOptions(); options.write_buffer_size = 64 * 1024; // very small @@ -6927,6 +7030,28 @@ TEST_F(DBTest, RowCache) { ASSERT_EQ(Get("foo"), "bar"); ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1); ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1); + + // Also test non-OK cache insertion (would be ASAN failure on memory leak) + class FailInsertionCache : public CacheWrapper { + public: + using CacheWrapper::CacheWrapper; + const char* Name() const override { return "FailInsertionCache"; } + Status Insert(const Slice&, Cache::ObjectPtr, const CacheItemHelper*, + size_t, Handle** = nullptr, Priority = Priority::LOW, + const Slice& /*compressed*/ = Slice(), + CompressionType /*type*/ = kNoCompression) override { + return Status::MemoryLimit(); + } + }; + options.row_cache = std::make_shared(options.row_cache); + ASSERT_OK(options.statistics->Reset()); + Reopen(options); + + ASSERT_EQ(Get("foo"), "bar"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1); + ASSERT_EQ(Get("foo"), "bar"); + // Test condition requires row cache insertion to fail + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 2); } TEST_F(DBTest, PinnableSliceAndRowCache) { @@ -7185,14 +7310,14 @@ TEST_F(DBTest, CreationTimeOfOldestFile) { int idx = 0; int64_t time_1 = 0; - env_->GetCurrentTime(&time_1); + ASSERT_OK(env_->GetCurrentTime(&time_1)); const uint64_t uint_time_1 = static_cast(time_1); // Add 50 hours env_->MockSleepForSeconds(50 * 60 * 60); int64_t time_2 = 0; - env_->GetCurrentTime(&time_2); + ASSERT_OK(env_->GetCurrentTime(&time_2)); const uint64_t uint_time_2 = static_cast(time_2); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( diff --git a/db/db_test2.cc b/db/db_test2.cc index 52d64a900..db879268c 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -266,7 +266,7 @@ TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) { ASSERT_OK(Put(1, "a", "begin")); ASSERT_OK(Put(1, "z", "end")); ASSERT_OK(Flush(1)); - TryReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); std::string value; value = Get(1, "a"); @@ -357,10 +357,10 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) { // are newer CFs created. flush_listener->expected_flush_reason = FlushReason::kManualFlush; ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); - Flush(3); + ASSERT_OK(Flush(3)); ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); - Flush(0); + ASSERT_OK(Flush(0)); ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), static_cast(1)); ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), @@ -1805,6 +1805,7 @@ TEST_P(CompressionFailuresTest, CompressionFailures) { ASSERT_EQ(key_value_written[key], value); key_value_written.erase(key); } + ASSERT_OK(db_iter->status()); ASSERT_EQ(0, key_value_written.size()); } else if (compression_failure_type_ == kTestDecompressionFail) { ASSERT_EQ(std::string(s.getState()), @@ -2068,7 +2069,7 @@ class PinL0IndexAndFilterBlocksTest // reset block cache table_options.block_cache = NewLRUCache(64 * 1024); options->table_factory.reset(NewBlockBasedTableFactory(table_options)); - TryReopenWithColumnFamilies({"default", "pikachu"}, *options); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, *options)); // create new table at L0 ASSERT_OK(Put(1, "a2", "begin2")); ASSERT_OK(Put(1, "z2", "end2")); @@ -2188,7 +2189,7 @@ TEST_P(PinL0IndexAndFilterBlocksTest, DisablePrefetchingNonL0IndexAndFilter) { // Reopen database. If max_open_files is set as -1, table readers will be // preloaded. This will trigger a BlockBasedTable::Open() and prefetch // L0 index and filter. Level 1's prefetching is disabled in DB::Open() - TryReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); @@ -3801,10 +3802,11 @@ TEST_F(DBTest2, MemtableOnlyIterator) { count++; } ASSERT_TRUE(!it->Valid()); + ASSERT_OK(it->status()); ASSERT_EQ(2, count); delete it; - Flush(1); + ASSERT_OK(Flush(1)); // After flushing // point lookups @@ -3859,6 +3861,17 @@ TEST_F(DBTest2, LowPriWrite) { int64_t* rate_bytes_per_sec = static_cast(arg); ASSERT_EQ(1024 * 1024, *rate_bytes_per_sec); }); + + // Make a trivial L5 for L0 to compact into. L6 will be large so debt ratio + // will not cause compaction pressure. + Random rnd(301); + ASSERT_OK(Put("", rnd.RandomString(102400))); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + ASSERT_OK(Put("", "")); + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + // Block compaction ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ {"DBTest.LowPriWrite:0", "DBImpl::BGWorkCompaction"}, @@ -3880,16 +3893,26 @@ TEST_F(DBTest2, LowPriWrite) { ASSERT_OK(Put("", "", wo)); ASSERT_EQ(1, rate_limit_count.load()); + wo.low_pri = true; + std::string big_value = std::string(1 * 1024 * 1024, 'x'); + ASSERT_OK(Put("", big_value, wo)); + ASSERT_LT(1, rate_limit_count.load()); + // Reset + rate_limit_count = 0; + wo.low_pri = false; + ASSERT_OK(Put("", big_value, wo)); + ASSERT_EQ(0, rate_limit_count.load()); + TEST_SYNC_POINT("DBTest.LowPriWrite:0"); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); ASSERT_OK(dbfull()->TEST_WaitForCompact()); wo.low_pri = true; ASSERT_OK(Put("", "", wo)); - ASSERT_EQ(1, rate_limit_count.load()); + ASSERT_EQ(0, rate_limit_count.load()); wo.low_pri = false; ASSERT_OK(Put("", "", wo)); - ASSERT_EQ(1, rate_limit_count.load()); + ASSERT_EQ(0, rate_limit_count.load()); } TEST_F(DBTest2, RateLimitedCompactionReads) { @@ -4112,7 +4135,7 @@ TEST_F(DBTest2, LiveFilesOmitObsoleteFiles) { ASSERT_OK(Put("key", "val")); FlushOptions flush_opts; flush_opts.wait = false; - db_->Flush(flush_opts); + ASSERT_OK(db_->Flush(flush_opts)); TEST_SYNC_POINT("DBTest2::LiveFilesOmitObsoleteFiles:FlushTriggered"); ASSERT_OK(db_->DisableFileDeletions()); @@ -4123,7 +4146,7 @@ TEST_F(DBTest2, LiveFilesOmitObsoleteFiles) { ASSERT_OK(env_->FileExists(LogFileName(dbname_, log_file->LogNumber()))); } - ASSERT_OK(db_->EnableFileDeletions()); + ASSERT_OK(db_->EnableFileDeletions(/*force=*/false)); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } @@ -6876,6 +6899,7 @@ TEST_F(DBTest2, LastLevelTemperatureUniversal) { TEST_F(DBTest2, LastLevelStatistics) { Options options = CurrentOptions(); options.bottommost_temperature = Temperature::kWarm; + options.default_temperature = Temperature::kHot; options.level0_file_num_compaction_trigger = 2; options.level_compaction_dynamic_level_bytes = true; options.statistics = CreateDBStatistics(); @@ -6889,6 +6913,10 @@ TEST_F(DBTest2, LastLevelStatistics) { ASSERT_GT(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_BYTES), 0); ASSERT_GT(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_BYTES), + options.statistics->getTickerCount(HOT_FILE_READ_BYTES)); + ASSERT_EQ(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT), + options.statistics->getTickerCount(HOT_FILE_READ_COUNT)); ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_BYTES), 0); ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_COUNT), 0); @@ -6899,6 +6927,10 @@ TEST_F(DBTest2, LastLevelStatistics) { ASSERT_OK(dbfull()->TEST_WaitForCompact()); ASSERT_EQ("bar", Get("bar")); + ASSERT_EQ(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_BYTES), + options.statistics->getTickerCount(HOT_FILE_READ_BYTES)); + ASSERT_EQ(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT), + options.statistics->getTickerCount(HOT_FILE_READ_COUNT)); ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_BYTES), options.statistics->getTickerCount(WARM_FILE_READ_BYTES)); ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_COUNT), @@ -6919,6 +6951,30 @@ TEST_F(DBTest2, LastLevelStatistics) { pre_bytes); ASSERT_GT(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT), pre_count); + ASSERT_EQ(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_BYTES), + options.statistics->getTickerCount(HOT_FILE_READ_BYTES)); + ASSERT_EQ(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT), + options.statistics->getTickerCount(HOT_FILE_READ_COUNT)); + ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_BYTES), + options.statistics->getTickerCount(WARM_FILE_READ_BYTES)); + ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_COUNT), + options.statistics->getTickerCount(WARM_FILE_READ_COUNT)); + + // Not a realistic setting to make last level kWarm and default temp kCold. + // This is just for testing default temp can be reset on reopen while the + // last level temp is consistent across DB reopen because those file's temp + // are persisted in manifest. + options.default_temperature = Temperature::kCold; + ASSERT_OK(options.statistics->Reset()); + Reopen(options); + ASSERT_EQ("bar", Get("bar")); + + ASSERT_EQ(0, options.statistics->getTickerCount(HOT_FILE_READ_BYTES)); + + ASSERT_EQ(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_BYTES), + options.statistics->getTickerCount(COLD_FILE_READ_BYTES)); + ASSERT_EQ(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT), + options.statistics->getTickerCount(COLD_FILE_READ_COUNT)); ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_BYTES), options.statistics->getTickerCount(WARM_FILE_READ_BYTES)); ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_COUNT), @@ -7555,6 +7611,7 @@ TEST_F(DBTest2, BestEffortsRecoveryWithSstUniqueIdVerification) { ASSERT_EQ(std::to_string(cnt), it->key()); ASSERT_EQ(expected_v, it->value()); } + EXPECT_OK(it->status()); ASSERT_EQ(expected_count, cnt); }; @@ -7670,6 +7727,41 @@ TEST_F(DBTest2, GetLatestSeqAndTsForKey) { ASSERT_EQ(0, options.statistics->getTickerCount(GET_HIT_L0)); } +#if defined(ZSTD_ADVANCED) +TEST_F(DBTest2, ZSTDChecksum) { + // Verify that corruption during decompression is caught. + Options options = CurrentOptions(); + options.create_if_missing = true; + options.compression = kZSTD; + options.compression_opts.max_compressed_bytes_per_kb = 1024; + options.compression_opts.checksum = true; + DestroyAndReopen(options); + Random rnd(33); + ASSERT_OK(Put(Key(0), rnd.RandomString(4 << 10))); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::WriteBlock:TamperWithCompressedData", + [&](void* arg) { + std::string* output = static_cast(arg); + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames + // Checksum is the last 4 bytes, corrupting that part in unit test is + // more controllable. + output->data()[output->size() - 1]++; + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Flush()); + PinnableSlice val; + Status s = Get(Key(0), &val); + ASSERT_TRUE(s.IsCorruption()); + + // Corruption caught during flush. + options.paranoid_file_checks = true; + DestroyAndReopen(options); + ASSERT_OK(Put(Key(0), rnd.RandomString(4 << 10))); + s = Flush(); + ASSERT_TRUE(s.IsCorruption()); +} +#endif + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/db_test_util.cc b/db/db_test_util.cc index 5a64b2f3f..3fb457676 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -259,7 +259,7 @@ bool DBTestBase::ChangeFilterOptions() { auto options = CurrentOptions(); options.create_if_missing = true; - TryReopen(options); + EXPECT_OK(TryReopen(options)); return true; } @@ -270,34 +270,34 @@ bool DBTestBase::ChangeOptionsForFileIngestionTest() { Destroy(last_options_); auto options = CurrentOptions(); options.create_if_missing = true; - TryReopen(options); + EXPECT_OK(TryReopen(options)); return true; } else if (option_config_ == kUniversalCompaction) { option_config_ = kUniversalCompactionMultiLevel; Destroy(last_options_); auto options = CurrentOptions(); options.create_if_missing = true; - TryReopen(options); + EXPECT_OK(TryReopen(options)); return true; } else if (option_config_ == kUniversalCompactionMultiLevel) { option_config_ = kLevelSubcompactions; Destroy(last_options_); auto options = CurrentOptions(); assert(options.max_subcompactions > 1); - TryReopen(options); + EXPECT_OK(TryReopen(options)); return true; } else if (option_config_ == kLevelSubcompactions) { option_config_ = kUniversalSubcompactions; Destroy(last_options_); auto options = CurrentOptions(); assert(options.max_subcompactions > 1); - TryReopen(options); + EXPECT_OK(TryReopen(options)); return true; } else if (option_config_ == kUniversalSubcompactions) { option_config_ = kDirectIO; Destroy(last_options_); auto options = CurrentOptions(); - TryReopen(options); + EXPECT_OK(TryReopen(options)); return true; } else { return false; @@ -699,6 +699,7 @@ void DBTestBase::Destroy(const Options& options, bool delete_cf_paths) { } Status DBTestBase::ReadOnlyReopen(const Options& options) { + Close(); MaybeInstallTimeElapseOnlySleep(options); return DB::OpenForReadOnly(options, dbname_, &db_); } @@ -942,6 +943,7 @@ std::string DBTestBase::Contents(int cf) { EXPECT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]); matched++; } + EXPECT_OK(iter->status()); EXPECT_EQ(matched, forward.size()); delete iter; @@ -1364,6 +1366,7 @@ std::string DBTestBase::IterStatus(Iterator* iter) { if (iter->Valid()) { result = iter->key().ToString() + "->" + iter->value().ToString(); } else { + EXPECT_OK(iter->status()); result = "(invalid)"; } return result; @@ -1582,6 +1585,7 @@ void DBTestBase::VerifyDBFromMap(std::map true_data, iter_cnt++; total_reads++; } + ASSERT_OK(iter->status()); ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / " << true_data.size(); delete iter; @@ -1605,6 +1609,7 @@ void DBTestBase::VerifyDBFromMap(std::map true_data, iter_cnt++; total_reads++; } + ASSERT_OK(iter->status()); ASSERT_EQ(data_rev, true_data.rend()) << iter_cnt << " / " << true_data.size(); @@ -1716,12 +1721,12 @@ TargetCacheChargeTrackingCache::TargetCacheChargeTrackingCache( cache_charge_increments_sum_(0) {} template -Status TargetCacheChargeTrackingCache::Insert(const Slice& key, - ObjectPtr value, - const CacheItemHelper* helper, - size_t charge, Handle** handle, - Priority priority) { - Status s = target_->Insert(key, value, helper, charge, handle, priority); +Status TargetCacheChargeTrackingCache::Insert( + const Slice& key, ObjectPtr value, const CacheItemHelper* helper, + size_t charge, Handle** handle, Priority priority, const Slice& compressed, + CompressionType type) { + Status s = target_->Insert(key, value, helper, charge, handle, priority, + compressed, type); if (helper == kCrmHelper) { if (last_peak_tracked_) { cache_charge_peak_ = 0; diff --git a/db/db_test_util.h b/db/db_test_util.h index 52e856cb3..023784f61 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -233,6 +233,7 @@ class SpecialEnv : public EnvWrapper { size_t GetUniqueId(char* id, size_t max_size) const override { return base_->GetUniqueId(id, max_size); } + uint64_t GetFileSize() final { return base_->GetFileSize(); } }; class ManifestFile : public WritableFile { public: @@ -345,6 +346,7 @@ class SpecialEnv : public EnvWrapper { Status Allocate(uint64_t offset, uint64_t len) override { return base_->Allocate(offset, len); } + uint64_t GetFileSize() final { return base_->GetFileSize(); } private: SpecialEnv* env_; @@ -936,8 +938,9 @@ class TargetCacheChargeTrackingCache : public CacheWrapper { Status Insert(const Slice& key, ObjectPtr value, const CacheItemHelper* helper, size_t charge, - Handle** handle = nullptr, - Priority priority = Priority::LOW) override; + Handle** handle = nullptr, Priority priority = Priority::LOW, + const Slice& compressed = Slice(), + CompressionType type = kNoCompression) override; using Cache::Release; bool Release(Handle* handle, bool erase_if_last_ref = false) override; diff --git a/db/db_universal_compaction_test.cc b/db/db_universal_compaction_test.cc index 84f01b3d1..5c10cdaac 100644 --- a/db/db_universal_compaction_test.cc +++ b/db/db_universal_compaction_test.cc @@ -7,9 +7,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include + #include "db/db_test_util.h" #include "port/stack_trace.h" #include "rocksdb/utilities/table_properties_collectors.h" +#include "test_util/mock_time_env.h" #include "test_util/sync_point.h" #include "test_util/testutil.h" #include "util/random.h" @@ -1469,6 +1472,7 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) { keys_in_db.append(iter->key().ToString()); keys_in_db.push_back(','); } + EXPECT_OK(iter->status()); delete iter; std::string expected_keys; @@ -2145,7 +2149,19 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompactionDefault) { options.ttl = 60 * 24 * 60 * 60; options.compaction_filter = nullptr; Reopen(options); - ASSERT_EQ(60 * 24 * 60 * 60, + ASSERT_EQ(30 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); + + options.periodic_compaction_seconds = 45 * 24 * 60 * 60; + options.ttl = 50 * 24 * 60 * 60; + Reopen(options); + ASSERT_EQ(45 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); + + options.periodic_compaction_seconds = 0; + options.ttl = 50 * 24 * 60 * 60; + Reopen(options); + ASSERT_EQ(50 * 24 * 60 * 60, dbfull()->GetOptions().periodic_compaction_seconds); } @@ -2217,6 +2233,141 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) { ASSERT_EQ(4, output_level); } +TEST_F(DBTestUniversalCompaction2, PeriodicCompactionOffpeak) { + constexpr int kSecondsPerDay = 86400; + constexpr int kSecondsPerHour = 3600; + constexpr int kSecondsPerMinute = 60; + + Options opts = CurrentOptions(); + opts.compaction_style = kCompactionStyleUniversal; + opts.level0_file_num_compaction_trigger = 10; + opts.max_open_files = -1; + opts.compaction_options_universal.size_ratio = 10; + opts.compaction_options_universal.min_merge_width = 2; + opts.compaction_options_universal.max_size_amplification_percent = 200; + opts.periodic_compaction_seconds = 5 * kSecondsPerDay; // 5 days + opts.num_levels = 5; + + // Just to add some extra random days to current time + Random rnd(test::RandomSeed()); + int days = rnd.Uniform(100); + + int periodic_compactions = 0; + int start_level = -1; + int output_level = -1; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionPicker::PickPeriodicCompaction:Return", + [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(arg != nullptr); + ASSERT_TRUE(compaction->compaction_reason() == + CompactionReason::kPeriodicCompaction); + start_level = compaction->start_level(); + output_level = compaction->output_level(); + periodic_compactions++; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + for (std::string preset_offpeak_time : {"", "00:30-04:30", "10:30-02:30"}) { + SCOPED_TRACE("preset_offpeak_time=" + preset_offpeak_time); + for (std::string new_offpeak_time : {"", "23:30-02:30"}) { + SCOPED_TRACE("new_offpeak_time=" + new_offpeak_time); + std::vector> times_to_test = { + {0, 0}, {2, 30}, {3, 15}, {5, 10}, {13, 30}, {23, 30}}; + for (std::pair now : times_to_test) { + int now_hour = now.first; + int now_minute = now.second; + SCOPED_TRACE("now=" + std::to_string(now_hour) + ":" + + std::to_string(now_minute)); + + auto mock_clock = + std::make_shared(env_->GetSystemClock()); + auto mock_env = std::make_unique(env_, mock_clock); + opts.env = mock_env.get(); + mock_clock->SetCurrentTime(days * kSecondsPerDay + + now_hour * kSecondsPerHour + + now_minute * kSecondsPerMinute); + opts.daily_offpeak_time_utc = preset_offpeak_time; + Reopen(opts); + + ASSERT_OK(Put("foo", "bar1")); + ASSERT_OK(Flush()); + ASSERT_EQ(0, periodic_compactions); + + // Move clock forward by 8 hours. There should be no periodic + // compaction, yet. + mock_clock->MockSleepForSeconds(8 * kSecondsPerHour); + ASSERT_OK(Put("foo", "bar2")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, periodic_compactions); + + // Move clock forward by 4 days + mock_clock->MockSleepForSeconds(4 * kSecondsPerDay); + ASSERT_OK(Put("foo", "bar3")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + int64_t mock_now; + ASSERT_OK(mock_clock->GetCurrentTime(&mock_now)); + + auto offpeak_time_info = + dbfull()->GetVersionSet()->offpeak_time_option().GetOffpeakTimeInfo( + mock_now); + // At this point, the first file is 4 days and 8 hours old. + // If it's offpeak now and the file is expected to expire before the + // next offpeak starts + if (offpeak_time_info.is_now_offpeak && + offpeak_time_info.seconds_till_next_offpeak_start / + kSecondsPerHour > + 16) { + ASSERT_EQ(1, periodic_compactions); + } else { + ASSERT_EQ(0, periodic_compactions); + // Change offpeak option by SetDBOption() + if (preset_offpeak_time != new_offpeak_time) { + ASSERT_OK(dbfull()->SetDBOptions( + {{"daily_offpeak_time_utc", new_offpeak_time}})); + ASSERT_OK(Put("foo", "bar4")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + offpeak_time_info = dbfull() + ->GetVersionSet() + ->offpeak_time_option() + .GetOffpeakTimeInfo(mock_now); + // if the first file is now eligible to be picked up + if (offpeak_time_info.is_now_offpeak && + offpeak_time_info.seconds_till_next_offpeak_start / + kSecondsPerHour > + 16) { + ASSERT_OK(Put("foo", "bar5")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(1, periodic_compactions); + } + } + + // If the file has not been picked up yet (no offpeak set, or offpeak + // set but then unset before the file becomes eligible) + if (periodic_compactions == 0) { + // move clock forward by one more day + mock_clock->MockSleepForSeconds(1 * kSecondsPerDay); + ASSERT_OK(Put("foo", "bar6")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + } + ASSERT_EQ(1, periodic_compactions); + ASSERT_EQ(0, start_level); + ASSERT_EQ(4, output_level); + Destroy(opts); + + periodic_compactions = 0; + } + } + } +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/db_wal_test.cc b/db/db_wal_test.cc index 232d32972..fbc01131e 100644 --- a/db/db_wal_test.cc +++ b/db/db_wal_test.cc @@ -14,6 +14,7 @@ #include "port/stack_trace.h" #include "rocksdb/file_system.h" #include "test_util/sync_point.h" +#include "util/udt_util.h" #include "utilities/fault_injection_env.h" #include "utilities/fault_injection_fs.h" @@ -318,24 +319,25 @@ class DBWALTestWithTimestamp DBWALTestWithTimestamp() : DBBasicTestWithTimestampBase("db_wal_test_with_timestamp") {} - void SetUp() override { - persist_udt_ = test::ShouldPersistUDT(GetParam()); - DBBasicTestWithTimestampBase::SetUp(); - } - - Status CreateAndReopenWithCFWithTs(const std::vector& cfs, - const Options& options, - bool avoid_flush_during_recovery = false) { - CreateColumnFamilies(cfs, options); - return ReopenColumnFamiliesWithTs(cfs, options, + Status CreateAndReopenWithTs(const std::vector& cfs, + const Options& ts_options, bool persist_udt, + bool avoid_flush_during_recovery = false) { + Options default_options = CurrentOptions(); + default_options.allow_concurrent_memtable_write = + persist_udt ? true : false; + DestroyAndReopen(default_options); + CreateColumnFamilies(cfs, ts_options); + return ReopenColumnFamiliesWithTs(cfs, ts_options, persist_udt, avoid_flush_during_recovery); } Status ReopenColumnFamiliesWithTs(const std::vector& cfs, - Options ts_options, + Options ts_options, bool persist_udt, bool avoid_flush_during_recovery = false) { Options default_options = CurrentOptions(); default_options.create_if_missing = false; + default_options.allow_concurrent_memtable_write = + persist_udt ? true : false; default_options.avoid_flush_during_recovery = avoid_flush_during_recovery; ts_options.create_if_missing = false; @@ -363,19 +365,15 @@ class DBWALTestWithTimestamp ASSERT_EQ(expected_value, actual_value); ASSERT_EQ(expected_ts, actual_ts); } - - protected: - bool persist_udt_; }; TEST_P(DBWALTestWithTimestamp, RecoverAndNoFlush) { // Set up the option that enables user defined timestmp size. - std::string ts1 = Timestamp(1, 0); - const size_t kTimestampSize = ts1.size(); - TestComparator test_cmp(kTimestampSize); + std::string ts1; + PutFixed64(&ts1, 1); Options ts_options; ts_options.create_if_missing = true; - ts_options.comparator = &test_cmp; + ts_options.comparator = test::BytewiseComparatorWithU64TsWrapper(); // Test that user-defined timestamps are recovered from WAL regardless of // the value of this flag because UDTs are saved in WAL nonetheless. // We however need to explicitly disable flush during recovery by setting @@ -383,20 +381,22 @@ TEST_P(DBWALTestWithTimestamp, RecoverAndNoFlush) { // stripped when the `persist_user_defined_timestamps` flag is false, so that // all written timestamps are available for testing user-defined time travel // read. - ts_options.persist_user_defined_timestamps = persist_udt_; + bool persist_udt = test::ShouldPersistUDT(GetParam()); + ts_options.persist_user_defined_timestamps = persist_udt; bool avoid_flush_during_recovery = true; + std::string full_history_ts_low; ReadOptions read_opts; do { Slice ts_slice = ts1; read_opts.timestamp = &ts_slice; - ASSERT_OK(CreateAndReopenWithCFWithTs({"pikachu"}, ts_options, - avoid_flush_during_recovery)); + ASSERT_OK(CreateAndReopenWithTs({"pikachu"}, ts_options, persist_udt, + avoid_flush_during_recovery)); ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 0U); ASSERT_OK(Put(1, "foo", ts1, "v1")); ASSERT_OK(Put(1, "baz", ts1, "v5")); - ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options, + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options, persist_udt, avoid_flush_during_recovery)); ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 0U); // Do a timestamped read with ts1 after second reopen. @@ -405,17 +405,24 @@ TEST_P(DBWALTestWithTimestamp, RecoverAndNoFlush) { // Write more value versions for key "foo" and "bar" before and after second // reopen. - std::string ts2 = Timestamp(2, 0); + std::string ts2; + PutFixed64(&ts2, 2); ASSERT_OK(Put(1, "bar", ts2, "v2")); ASSERT_OK(Put(1, "foo", ts2, "v3")); - ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options, + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options, persist_udt, avoid_flush_during_recovery)); ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 0U); - std::string ts3 = Timestamp(3, 0); + std::string ts3; + PutFixed64(&ts3, 3); ASSERT_OK(Put(1, "foo", ts3, "v4")); + // All the key value pairs available for read: + // "foo" -> [(ts1, "v1"), (ts2, "v3"), (ts3, "v4")] + // "bar" -> [(ts2, "v2")] + // "baz" -> [(ts1, "v5")] // Do a timestamped read with ts1 after third reopen. + // read_opts.timestamp is set to ts1 for below reads CheckGet(read_opts, 1, "foo", "v1", ts1); std::string value; ASSERT_TRUE(db_->Get(read_opts, handles_[1], "bar", &value).IsNotFound()); @@ -423,92 +430,121 @@ TEST_P(DBWALTestWithTimestamp, RecoverAndNoFlush) { // Do a timestamped read with ts2 after third reopen. ts_slice = ts2; + // read_opts.timestamp is set to ts2 for below reads. CheckGet(read_opts, 1, "foo", "v3", ts2); CheckGet(read_opts, 1, "bar", "v2", ts2); CheckGet(read_opts, 1, "baz", "v5", ts1); // Do a timestamped read with ts3 after third reopen. ts_slice = ts3; + // read_opts.timestamp is set to ts3 for below reads. CheckGet(read_opts, 1, "foo", "v4", ts3); CheckGet(read_opts, 1, "bar", "v2", ts2); CheckGet(read_opts, 1, "baz", "v5", ts1); + ASSERT_OK(db_->GetFullHistoryTsLow(handles_[1], &full_history_ts_low)); + ASSERT_TRUE(full_history_ts_low.empty()); } while (ChangeWalOptions()); } -TEST_P(DBWALTestWithTimestamp, RecoverInconsistentTimestamp) { - // Set up the option that enables user defined timestmp size. - std::string ts = Timestamp(1, 0); - const size_t kTimestampSize = ts.size(); - TestComparator test_cmp(kTimestampSize); - Options ts_options; - ts_options.create_if_missing = true; - ts_options.comparator = &test_cmp; - ts_options.persist_user_defined_timestamps = persist_udt_; - - ASSERT_OK(CreateAndReopenWithCFWithTs({"pikachu"}, ts_options)); - ASSERT_OK(Put(1, "foo", ts, "v1")); - ASSERT_OK(Put(1, "baz", ts, "v5")); - - // In real use cases, switching to a different user comparator is prohibited - // by a sanity check during DB open that does a user comparator name - // comparison. This test mocked and bypassed that sanity check because the - // before and after user comparator are both named "TestComparator". This is - // to test the user-defined timestamp recovery logic for WAL files have - // the intended consistency check. - // `HandleWriteBatchTimestampSizeDifference` in udt_util.h has more details. - TestComparator diff_test_cmp(kTimestampSize + 1); - ts_options.comparator = &diff_test_cmp; - ASSERT_TRUE( - ReopenColumnFamiliesWithTs({"pikachu"}, ts_options).IsInvalidArgument()); -} - TEST_P(DBWALTestWithTimestamp, RecoverAndFlush) { // Set up the option that enables user defined timestamp size. - std::string min_ts = Timestamp(0, 0); - std::string write_ts = Timestamp(1, 0); - const size_t kTimestampSize = write_ts.size(); - TestComparator test_cmp(kTimestampSize); + std::string min_ts; + std::string write_ts; + PutFixed64(&min_ts, 0); + PutFixed64(&write_ts, 1); Options ts_options; ts_options.create_if_missing = true; - ts_options.comparator = &test_cmp; - ts_options.persist_user_defined_timestamps = persist_udt_; + ts_options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + bool persist_udt = test::ShouldPersistUDT(GetParam()); + ts_options.persist_user_defined_timestamps = persist_udt; std::string smallest_ukey_without_ts = "baz"; std::string largest_ukey_without_ts = "foo"; - ASSERT_OK(CreateAndReopenWithCFWithTs({"pikachu"}, ts_options)); + ASSERT_OK(CreateAndReopenWithTs({"pikachu"}, ts_options, persist_udt)); // No flush, no sst files, because of no data. ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 0U); ASSERT_OK(Put(1, largest_ukey_without_ts, write_ts, "v1")); ASSERT_OK(Put(1, smallest_ukey_without_ts, write_ts, "v5")); - ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options)); + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options, persist_udt)); // Memtable recovered from WAL flushed because `avoid_flush_during_recovery` // defaults to false, created one L0 file. ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 1U); std::vector> level_to_files; dbfull()->TEST_GetFilesMetaData(handles_[1], &level_to_files); + std::string full_history_ts_low; + ASSERT_OK(db_->GetFullHistoryTsLow(handles_[1], &full_history_ts_low)); ASSERT_GT(level_to_files.size(), 1); // L0 only has one SST file. ASSERT_EQ(level_to_files[0].size(), 1); auto meta = level_to_files[0][0]; - if (persist_udt_) { + if (persist_udt) { ASSERT_EQ(smallest_ukey_without_ts + write_ts, meta.smallest.user_key()); ASSERT_EQ(largest_ukey_without_ts + write_ts, meta.largest.user_key()); + ASSERT_TRUE(full_history_ts_low.empty()); } else { ASSERT_EQ(smallest_ukey_without_ts + min_ts, meta.smallest.user_key()); ASSERT_EQ(largest_ukey_without_ts + min_ts, meta.largest.user_key()); + std::string effective_cutoff; + Slice write_ts_slice = write_ts; + GetFullHistoryTsLowFromU64CutoffTs(&write_ts_slice, &effective_cutoff); + ASSERT_EQ(effective_cutoff, full_history_ts_low); } } // Param 0: test mode for the user-defined timestamp feature INSTANTIATE_TEST_CASE_P( - DBWALTestWithTimestamp, DBWALTestWithTimestamp, + P, DBWALTestWithTimestamp, ::testing::Values( test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp, test::UserDefinedTimestampTestMode::kNormal)); +TEST_F(DBWALTestWithTimestamp, EnableDisableUDT) { + Options options; + options.create_if_missing = true; + options.comparator = BytewiseComparator(); + bool avoid_flush_during_recovery = true; + ASSERT_OK(CreateAndReopenWithTs({"pikachu"}, options, true /* persist_udt */, + avoid_flush_during_recovery)); + + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "foo", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "baz", "v5")); + + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.persist_user_defined_timestamps = false; + // Test handle timestamp size inconsistency in WAL when enabling user-defined + // timestamps. + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, options, + false /* persist_udt */, + avoid_flush_during_recovery)); + + std::string ts; + PutFixed64(&ts, 0); + Slice ts_slice = ts; + ReadOptions read_opts; + read_opts.timestamp = &ts_slice; + // Pre-existing entries are treated as if they have the min timestamp. + CheckGet(read_opts, 1, "foo", "v1", ts); + CheckGet(read_opts, 1, "baz", "v5", ts); + ts.clear(); + PutFixed64(&ts, 1); + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "foo", ts, "v2")); + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "baz", ts, "v6")); + CheckGet(read_opts, 1, "foo", "v2", ts); + CheckGet(read_opts, 1, "baz", "v6", ts); + + options.comparator = BytewiseComparator(); + // Open the column family again with the UDT feature disabled. Test handle + // timestamp size inconsistency in WAL when disabling user-defined timestamps + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, options, + true /* persist_udt */, + avoid_flush_during_recovery)); + ASSERT_EQ("v2", Get(1, "foo")); + ASSERT_EQ("v6", Get(1, "baz")); +} + TEST_F(DBWALTest, RecoverWithTableHandle) { do { Options options = CurrentOptions(); @@ -1159,7 +1195,7 @@ TEST_F(DBWALTest, DISABLED_FullPurgePreservesLogPendingReuse) { ROCKSDB_NAMESPACE::port::Thread thread([&]() { TEST_SYNC_POINT( "DBWALTest::FullPurgePreservesLogPendingReuse:PreFullPurge"); - ASSERT_OK(db_->EnableFileDeletions(true)); + ASSERT_OK(db_->EnableFileDeletions(/*force=*/true)); TEST_SYNC_POINT( "DBWALTest::FullPurgePreservesLogPendingReuse:PostFullPurge"); }); @@ -1505,7 +1541,9 @@ class RecoveryTestHelper { test->dbname_, &db_options, file_options, table_cache.get(), &write_buffer_manager, &write_controller, /*block_cache_tracer=*/nullptr, - /*io_tracer=*/nullptr, /*db_id*/ "", /*db_session_id*/ "")); + /*io_tracer=*/nullptr, /*db_id=*/"", /*db_session_id=*/"", + options.daily_offpeak_time_utc, + /*error_handler=*/nullptr)); wal_manager.reset( new WalManager(db_options, file_options, /*io_tracer=*/nullptr)); @@ -2192,6 +2230,7 @@ TEST_P(DBWALTestWithParamsVaryingRecoveryMode, data.push_back( std::make_pair(iter->key().ToString(), iter->value().ToString())); } + EXPECT_OK(iter->status()); delete iter; return data; }; diff --git a/db/db_with_timestamp_basic_test.cc b/db/db_with_timestamp_basic_test.cc index bd82e49e0..3214faae9 100644 --- a/db/db_with_timestamp_basic_test.cc +++ b/db/db_with_timestamp_basic_test.cc @@ -459,6 +459,13 @@ TEST_F(DBBasicTestWithTimestamp, GetApproximateSizes) { db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); ASSERT_GT(size, 0); + uint64_t total_mem_count; + uint64_t total_mem_size; + db_->GetApproximateMemTableStats(default_cf, r, &total_mem_count, + &total_mem_size); + ASSERT_GT(total_mem_count, 0); + ASSERT_GT(total_mem_size, 0); + // Should exclude end key start = Key(900); end = Key(1000); @@ -518,6 +525,7 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) { CheckIterUserEntry(it.get(), Key1(key), kTypeValue, "value" + std::to_string(i), write_timestamps[i]); } + ASSERT_OK(it->status()); ASSERT_EQ(static_cast(kMaxKey) - start_keys[i] + 1, count); // SeekToFirst()/SeekToLast() with lower/upper bounds. @@ -537,6 +545,7 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) { CheckIterUserEntry(it.get(), Key1(key), kTypeValue, "value" + std::to_string(i), write_timestamps[i]); } + ASSERT_OK(it->status()); ASSERT_EQ(r - std::max(l, start_keys[i]), count); for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0; @@ -544,6 +553,7 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) { CheckIterUserEntry(it.get(), Key1(key - 1), kTypeValue, "value" + std::to_string(i), write_timestamps[i]); } + ASSERT_OK(it->status()); l += (kMaxKey / 100); r -= (kMaxKey / 100); } @@ -726,6 +736,7 @@ TEST_P(DBBasicTestWithTimestampTableOptions, GetAndMultiGet) { ASSERT_EQ(it->value(), value_from_get); ASSERT_EQ(Timestamp(1, 0), timestamp); } + ASSERT_OK(it->status()); // verify MultiGet() constexpr uint64_t step = 2; @@ -1058,6 +1069,7 @@ TEST_F(DBBasicTestWithTimestamp, SimpleForwardIterateLowerTsBound) { write_timestamps[i - 1]); } } + ASSERT_OK(it->status()); size_t expected_count = kMaxKey + 1; ASSERT_EQ(expected_count, count); } @@ -1136,6 +1148,7 @@ TEST_F(DBBasicTestWithTimestamp, BackwardIterateLowerTsBound) { write_timestamps[1]); } } + ASSERT_OK(it->status()); size_t expected_count = kMaxKey + 1; ASSERT_EQ(expected_count, count); } @@ -1166,6 +1179,7 @@ TEST_F(DBBasicTestWithTimestamp, BackwardIterateLowerTsBound) { CheckIterEntry(it.get(), Key1(key), kTypeDeletionWithTimestamp, Slice(), write_timestamp); } + ASSERT_OK(it->status()); ASSERT_EQ(kMaxKey + 1, count); } Close(); @@ -1271,6 +1285,7 @@ TEST_F(DBBasicTestWithTimestamp, BackwardIterateLowerTsBound_Reseek) { CheckIterEntry(it.get(), "a", kTypeValue, "v" + std::to_string(4 + i), Timestamp(4 + i, 0)); } + ASSERT_OK(it->status()); } Close(); @@ -1610,6 +1625,219 @@ TEST_F(DBBasicTestWithTimestamp, MultiGetRangeFiltering) { Close(); } +TEST_F(DBBasicTestWithTimestamp, GetWithRowCache) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + LRUCacheOptions cache_options; + cache_options.capacity = 8192; + options.row_cache = cache_options.MakeSharedRowCache(); + + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + WriteOptions write_opts; + std::string ts_early = Timestamp(1, 0); + std::string ts_later = Timestamp(10, 0); + Slice ts_later_slice = ts_later; + + const Snapshot* snap_with_nothing = db_->GetSnapshot(); + ASSERT_OK(db_->Put(write_opts, "foo", ts_early, "bar")); + ASSERT_OK(db_->Put(write_opts, "foo2", ts_early, "bar2")); + ASSERT_OK(db_->Put(write_opts, "foo3", ts_early, "bar3")); + + const Snapshot* snap_with_foo = db_->GetSnapshot(); + ASSERT_OK(Flush()); + + ReadOptions read_opts; + read_opts.timestamp = &ts_later_slice; + + std::string read_value; + std::string read_ts; + Status s; + + int expected_hit_count = 0; + int expected_miss_count = 0; + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), expected_miss_count); + + { + read_opts.timestamp = nullptr; + s = db_->Get(read_opts, "foo", &read_value); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + } + + // Mix use of Get + { + read_opts.timestamp = &ts_later_slice; + + // Use Get without ts first, expect cache entry to store the correct ts + s = db_->Get(read_opts, "foo2", &read_value); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), + ++expected_miss_count); + ASSERT_EQ(read_value, "bar2"); + + s = db_->Get(read_opts, "foo2", &read_value, &read_ts); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), ++expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), expected_miss_count); + ASSERT_EQ(read_ts, ts_early); + ASSERT_EQ(read_value, "bar2"); + + // Use Get with ts first, expect the Get without ts can get correct record + s = db_->Get(read_opts, "foo3", &read_value, &read_ts); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), + ++expected_miss_count); + ASSERT_EQ(read_ts, ts_early); + ASSERT_EQ(read_value, "bar3"); + + s = db_->Get(read_opts, "foo3", &read_value); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), ++expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), expected_miss_count); + ASSERT_EQ(read_value, "bar3"); + } + + { + // Test with consecutive calls of Get with ts. + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), + ++expected_miss_count); + ASSERT_EQ(read_ts, ts_early); + ASSERT_EQ(read_value, "bar"); + + // Test repeated get on cache entry + for (int i = 0; i < 3; i++) { + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), + ++expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), + expected_miss_count); + ASSERT_EQ(read_ts, ts_early); + ASSERT_EQ(read_value, "bar"); + } + } + + { + std::string ts_nothing = Timestamp(0, 0); + Slice ts_nothing_slice = ts_nothing; + read_opts.timestamp = &ts_nothing_slice; + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), + ++expected_miss_count); + } + + { + read_opts.snapshot = snap_with_foo; + read_opts.timestamp = &ts_later_slice; + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), + ++expected_miss_count); + ASSERT_EQ(read_ts, ts_early); + ASSERT_EQ(read_value, "bar"); + + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), ++expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), expected_miss_count); + ASSERT_EQ(read_ts, ts_early); + ASSERT_EQ(read_value, "bar"); + } + + { + read_opts.snapshot = snap_with_nothing; + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), + ++expected_miss_count); + + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), expected_hit_count); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), + ++expected_miss_count); + } + + db_->ReleaseSnapshot(snap_with_nothing); + db_->ReleaseSnapshot(snap_with_foo); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, GetWithRowCacheMultiSST) { + BlockBasedTableOptions table_options; + table_options.block_size = 1; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + LRUCacheOptions cache_options; + cache_options.capacity = 8192; + options.row_cache = cache_options.MakeSharedRowCache(); + + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.merge_operator = MergeOperators::CreateStringAppendTESTOperator(); + options.disable_auto_compactions = true; + + DestroyAndReopen(options); + + std::string ts_early = Timestamp(1, 0); + std::string ts_later = Timestamp(10, 0); + Slice ts_later_slice = ts_later; + + ASSERT_OK(db_->Put(WriteOptions(), "foo", ts_early, "v1")); + ASSERT_OK(Flush()); + + ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily(); + ASSERT_OK( + db_->Merge(WriteOptions(), default_cf, "foo", Timestamp(2, 0), "v2")); + ASSERT_OK( + db_->Merge(WriteOptions(), default_cf, "foo", Timestamp(3, 0), "v3")); + ASSERT_OK(Flush()); + + ReadOptions read_opts; + read_opts.timestamp = &ts_later_slice; + + std::string read_value; + std::string read_ts; + Status s; + + { + // Since there are two SST files, will trigger the table lookup twice. + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 0); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 2); + ASSERT_EQ(read_ts, Timestamp(3, 0)); + ASSERT_EQ(read_value, "v1,v2,v3"); + + s = db_->Get(read_opts, "foo", &read_value, &read_ts); + ASSERT_OK(s); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 2); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 2); + ASSERT_EQ(read_ts, Timestamp(3, 0)); + ASSERT_EQ(read_value, "v1,v2,v3"); + } +} + TEST_P(DBBasicTestWithTimestampTableOptions, MultiGetPrefixFilter) { Options options = CurrentOptions(); options.env = env_; @@ -3039,6 +3267,7 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) { "value" + std::to_string(i), write_ts_list[i]); iter->Next(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); // Seek to kMinKey iter->Seek(Key1(kMinKey)); @@ -3046,6 +3275,7 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) { "value" + std::to_string(i), write_ts_list[i]); iter->Prev(); ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); } const std::vector targets = {kMinKey, kMinKey + 0x10, kMinKey + 0x100, kMaxKey}; @@ -3084,6 +3314,7 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) { ++expected_key; it->Next(); } + ASSERT_OK(it->status()); ASSERT_EQ(expected_ub - targets[j] + 1, count); count = 0; @@ -3102,6 +3333,7 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) { --expected_key; it->Prev(); } + ASSERT_OK(it->status()); ASSERT_EQ(targets[j] - std::max(expected_lb, kMinKey) + 1, count); } } @@ -3207,6 +3439,7 @@ TEST_P(DBBasicTestWithTsIterTombstones, IterWithDelete) { ASSERT_EQ(Key1(key), iter->key()); ASSERT_EQ("value1" + std::to_string(key), iter->value()); } + ASSERT_OK(iter->status()); ASSERT_EQ((kMaxKey - kMinKey + 1) / 2, count); } Close(); @@ -3289,15 +3522,18 @@ TEST_P(HandleFileBoundariesTest, ConfigurePersistUdt) { options.env = env_; // Write a timestamp that is not the min timestamp to help test the behavior // of flag `persist_user_defined_timestamps`. - std::string write_ts = Timestamp(1, 0); - std::string min_ts = Timestamp(0, 0); + std::string write_ts; + std::string min_ts; + PutFixed64(&write_ts, 1); + PutFixed64(&min_ts, 0); std::string smallest_ukey_without_ts = "bar"; std::string largest_ukey_without_ts = "foo"; - const size_t kTimestampSize = write_ts.size(); - TestComparator test_cmp(kTimestampSize); - options.comparator = &test_cmp; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); bool persist_udt = test::ShouldPersistUDT(GetParam()); options.persist_user_defined_timestamps = persist_udt; + if (!persist_udt) { + options.allow_concurrent_memtable_write = false; + } DestroyAndReopen(options); ASSERT_OK( @@ -3343,6 +3579,288 @@ INSTANTIATE_TEST_CASE_P( test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp, test::UserDefinedTimestampTestMode::kNormal)); +TEST_F(DBBasicTestWithTimestamp, EnableDisableUDT) { + Options options = CurrentOptions(); + options.env = env_; + // Create a column family without user-defined timestamps. + options.comparator = BytewiseComparator(); + options.persist_user_defined_timestamps = true; + DestroyAndReopen(options); + + // Create one SST file, its user keys have no user-defined timestamps. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "val1")); + ASSERT_OK(Flush(0)); + Close(); + + // Reopen the existing column family and enable user-defined timestamps + // feature for it. + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.persist_user_defined_timestamps = false; + options.allow_concurrent_memtable_write = false; + Reopen(options); + + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), "foo", &value).IsInvalidArgument()); + std::string read_ts; + PutFixed64(&read_ts, 0); + ReadOptions ropts; + Slice read_ts_slice = read_ts; + ropts.timestamp = &read_ts_slice; + std::string key_ts; + // Entries in pre-existing SST files are treated as if they have minimum + // user-defined timestamps. + ASSERT_OK(db_->Get(ropts, "foo", &value, &key_ts)); + ASSERT_EQ("val1", value); + ASSERT_EQ(read_ts, key_ts); + + // Do timestamped read / write. + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val2")); + read_ts.clear(); + PutFixed64(&read_ts, 1); + ASSERT_OK(db_->Get(ropts, "foo", &value, &key_ts)); + ASSERT_EQ("val2", value); + ASSERT_EQ(write_ts, key_ts); + // The user keys in this SST file don't have user-defined timestamps either, + // because `persist_user_defined_timestamps` flag is set to false. + ASSERT_OK(Flush(0)); + Close(); + + // Reopen the existing column family while disabling user-defined timestamps. + options.comparator = BytewiseComparator(); + Reopen(options); + + ASSERT_TRUE(db_->Get(ropts, "foo", &value).IsInvalidArgument()); + ASSERT_OK(db_->Get(ReadOptions(), "foo", &value)); + ASSERT_EQ("val2", value); + + // Continue to write / read the column family without user-defined timestamps. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "val3")); + ASSERT_OK(db_->Get(ReadOptions(), "foo", &value)); + ASSERT_EQ("val3", value); + Close(); +} + +// Tests that as long as the +// `ReadOptions.timestamp >= SuperVersion.full_history_ts_low` sanity check +// passes. The read will be consistent even if the column family's +// full_history_ts_low is concurrently increased and collapsed some history +// above `ReadOptions.timestamp`. +TEST_F(DBBasicTestWithTimestamp, + FullHistoryTsLowSanityCheckPassReadIsConsistent) { + Options options = CurrentOptions(); + options.env = env_; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + // Use UDT in memtable only feature for this test, so we can control that + // newly set `full_history_ts_low` collapse history when Flush happens. + options.persist_user_defined_timestamps = false; + options.allow_concurrent_memtable_write = false; + DestroyAndReopen(options); + std::string min_ts; + PutFixed64(&min_ts, 0); + + // Write two versions of the key (1, v1), (3, v3), and always read with + // timestamp 2. + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val1")); + + std::string read_ts; + PutFixed64(&read_ts, 2); + Slice read_ts_slice = read_ts; + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + + // First read, no full_history_ts_low set, sanity check pass. + std::string value; + std::string timestamp; + ASSERT_OK(db_->Get(read_opts, "foo", &value, ×tamp)); + ASSERT_EQ("val1", value); + ASSERT_EQ(write_ts, timestamp); + + std::string full_history_ts_low; + std::string marked_ts_low; + PutFixed64(&full_history_ts_low, 2); + marked_ts_low = full_history_ts_low; + ASSERT_OK(db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + full_history_ts_low)); + ASSERT_OK(Flush(0)); + + // Write the (3, v3) entry after flush, otherwise with UDT in memtable only + // the previous Flush(0) with full_history_ts_low = 2 will be postponed + // waiting for (3, v3) to expire too. + write_ts.clear(); + PutFixed64(&write_ts, 3); + ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val3")); + + // Second read: + // ReadOptions.timestamp(2) >= SuperVersion.full_history_ts_low(2), + // and ReadOptions.timestamp(2) >= ColumnFamilyData.full_history_ts_low(2). + // history below 2 is collapsed. Reading at 2 or above 2 is ok. + // Sanity check pass. Read return consistent value, but timestamp is already + // collapsed. + ASSERT_OK(db_->Get(read_opts, "foo", &value, ×tamp)); + ASSERT_EQ("val1", value); + ASSERT_EQ(min_ts, timestamp); + + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::GetImpl:AfterAcquireSv", [&](void* /*arg*/) { + // Concurrently increasing full_history_ts_low and flush to create a + // new SuperVersion + std::string current_ts_low; + ASSERT_OK(db_->GetFullHistoryTsLow(db_->DefaultColumnFamily(), + ¤t_ts_low)); + if (current_ts_low.empty() || current_ts_low != marked_ts_low) { + return; + } + full_history_ts_low.clear(); + PutFixed64(&full_history_ts_low, 4); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + full_history_ts_low)); + ASSERT_OK(Flush(0)); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + // Third read: + // ReadOptions.timestamp(2) >= SuperVersion.full_history_ts_low(2), + // but ReadOptions.timestamp(2) < ColumnFamilyData.full_history_ts_low(4). + // History below 4 is collapsed in the newly installed SuperVersion. But the + // SuperVersion attached to this read still has the history below 4 available. + // Sanity check pass. Read return consistent value, timestamp is collapsed. + ASSERT_OK(db_->Get(read_opts, "foo", &value, ×tamp)); + ASSERT_EQ("val1", value); + ASSERT_EQ(min_ts, timestamp); + + // Fourth read: + // ReadOptions.timestamp(2) < SuperVersion.full_history_ts_low(4). + // Sanity check fails. Had it succeeded, the read would return "v3", + // which is inconsistent. + ASSERT_TRUE( + db_->Get(read_opts, "foo", &value, ×tamp).IsInvalidArgument()); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +// Tests that in cases when +// `ReadOptions.timestamp >= SuperVersion.full_history_ts_low` sanity check +// fails. The referenced SuperVersion is dereferenced and cleaned up properly +// for all read APIs that involves this sanity check. +TEST_F(DBBasicTestWithTimestamp, FullHistoryTsLowSanityCheckFail) { + Options options = CurrentOptions(); + options.env = env_; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + // Use UDT in memtable only feature for this test, so we can control that + // newly set `full_history_ts_low` collapse history when Flush happens. + options.persist_user_defined_timestamps = false; + options.allow_concurrent_memtable_write = false; + DestroyAndReopen(options); + + ColumnFamilyHandle* handle2 = nullptr; + Status s = db_->CreateColumnFamily(options, "data", &handle2); + ASSERT_OK(s); + + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val1")); + ASSERT_OK(db_->Put(WriteOptions(), handle2, "foo", write_ts, "val1")); + + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 3); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + full_history_ts_low)); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handle2, full_history_ts_low)); + ASSERT_OK(Flush(0)); + ASSERT_OK(db_->Flush(FlushOptions(), handle2)); + + std::string read_ts; + PutFixed64(&read_ts, 2); + Slice read_ts_slice = read_ts; + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + + // Get() + std::string value; + ASSERT_TRUE(db_->Get(read_opts, "foo", &value).IsInvalidArgument()); + + // MultiGet() + std::vector cfhs = {db_->DefaultColumnFamily(), handle2}; + { + std::vector key_vals = {"foo", "foo"}; + std::vector keys; + std::vector values; + for (size_t j = 0; j < 2; ++j) { + keys.push_back(key_vals[j]); + } + + std::vector statuses = + db_->MultiGet(read_opts, cfhs, keys, &values); + for (auto status : statuses) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + } + + // MultiGet with only one column family + { + std::vector one_cfh = {db_->DefaultColumnFamily()}; + std::vector key_vals = {"foo"}; + std::vector keys; + std::vector values; + for (size_t j = 0; j < 1; ++j) { + keys.push_back(key_vals[j]); + } + + std::vector statuses = + db_->MultiGet(read_opts, one_cfh, keys, &values); + for (auto status : statuses) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + } + + // Overloaded version of MultiGet + ColumnFamilyHandle* column_families[] = {db_->DefaultColumnFamily(), handle2}; + { + Slice keys[] = {"foo", "foo"}; + PinnableSlice values[] = {PinnableSlice(), PinnableSlice()}; + Status statuses[] = {Status::OK(), Status::OK()}; + db_->MultiGet(read_opts, /*num_keys=*/2, &column_families[0], &keys[0], + &values[0], &statuses[0], /*sorted_input=*/false); + for (auto status : statuses) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + } + + // Overloaded versions of MultiGet with one column family + { + ColumnFamilyHandle* one_column_family[] = {db_->DefaultColumnFamily()}; + Slice keys[] = {"foo"}; + PinnableSlice values[] = {PinnableSlice()}; + Status statuses[] = {Status::OK()}; + db_->MultiGet(read_opts, /*num_keys=*/1, &one_column_family[0], &keys[0], + &values[0], &statuses[0], /*sorted_input=*/false); + for (auto status : statuses) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + } + + // NewIterator() + std::unique_ptr iter( + db_->NewIterator(read_opts, db_->DefaultColumnFamily())); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + std::unique_ptr iter2(db_->NewIterator(read_opts, handle2)); + ASSERT_TRUE(iter2->status().IsInvalidArgument()); + + // NewIterators() + std::vector iterators; + ASSERT_TRUE( + db_->NewIterators(read_opts, cfhs, &iterators).IsInvalidArgument()); + delete handle2; + Close(); +} + TEST_F(DBBasicTestWithTimestamp, GCPreserveRangeTombstoneWhenNoOrSmallFullHistoryLow) { Options options = CurrentOptions(); @@ -3541,6 +4059,7 @@ TEST_P(DBBasicTestWithTimestampTableOptions, DeleteRangeBaiscReadAndIterate) { ++expected; } } + ASSERT_OK(iter->status()); ASSERT_EQ(kNum, expected); expected = kNum / 2; @@ -3548,6 +4067,7 @@ TEST_P(DBBasicTestWithTimestampTableOptions, DeleteRangeBaiscReadAndIterate) { ASSERT_EQ(Key1(expected), iter->key()); ++expected; } + ASSERT_OK(iter->status()); ASSERT_EQ(kNum, expected); expected = kRangeBegin - 1; @@ -3555,6 +4075,7 @@ TEST_P(DBBasicTestWithTimestampTableOptions, DeleteRangeBaiscReadAndIterate) { ASSERT_EQ(Key1(expected), iter->key()); --expected; } + ASSERT_OK(iter->status()); ASSERT_EQ(-1, expected); read_ts = Timestamp(0, 0); @@ -3836,6 +4357,7 @@ TEST_F(DBBasicTestWithTimestamp, MergeBasic) { ASSERT_EQ(value, it->value()); ASSERT_EQ(write_ts_strs[i], it->timestamp()); } + EXPECT_OK(it->status()); ASSERT_EQ(kNumOfUniqKeys, key_int_val); key_int_val = kNumOfUniqKeys - 1; @@ -3847,6 +4369,7 @@ TEST_F(DBBasicTestWithTimestamp, MergeBasic) { ASSERT_EQ(value, it->value()); ASSERT_EQ(write_ts_strs[i], it->timestamp()); } + ASSERT_OK(it->status()); ASSERT_EQ(std::numeric_limits::max(), key_int_val); value_suffix = value_suffix + "." + std::to_string(i + 1); @@ -4083,4 +4606,4 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); RegisterCustomObjects(argc, argv); return RUN_ALL_TESTS(); -} +} \ No newline at end of file diff --git a/db/db_write_buffer_manager_test.cc b/db/db_write_buffer_manager_test.cc index 82704e194..eb33ec41e 100644 --- a/db/db_write_buffer_manager_test.cc +++ b/db/db_write_buffer_manager_test.cc @@ -42,10 +42,10 @@ TEST_P(DBWriteBufferManagerTest, SharedBufferAcrossCFs1) { CreateAndReopenWithCF({"cf1", "cf2", "cf3"}, options); ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); - Flush(3); + ASSERT_OK(Flush(3)); ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); - Flush(0); + ASSERT_OK(Flush(0)); // Write to "Default", "cf2" and "cf3". ASSERT_OK(Put(3, Key(1), DummyString(30000), wo)); @@ -84,10 +84,10 @@ TEST_P(DBWriteBufferManagerTest, SharedWriteBufferAcrossCFs2) { CreateAndReopenWithCF({"cf1", "cf2", "cf3"}, options); ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); - Flush(3); + ASSERT_OK(Flush(3)); ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); - Flush(0); + ASSERT_OK(Flush(0)); // Write to "Default", "cf2" and "cf3". No flush will be triggered. ASSERT_OK(Put(3, Key(1), DummyString(30000), wo)); @@ -471,10 +471,10 @@ TEST_P(DBWriteBufferManagerTest, MixedSlowDownOptionsSingleDB) { CreateAndReopenWithCF({"cf1", "cf2", "cf3"}, options); ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); - Flush(3); + ASSERT_OK(Flush(3)); ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); - Flush(0); + ASSERT_OK(Flush(0)); // Write to "Default", "cf2" and "cf3". No flush will be triggered. ASSERT_OK(Put(3, Key(1), DummyString(30000), wo)); diff --git a/db/db_write_test.cc b/db/db_write_test.cc index d1e3d53f6..59c26eaaa 100644 --- a/db/db_write_test.cc +++ b/db/db_write_test.cc @@ -495,7 +495,7 @@ TEST_P(DBWriteTest, UnflushedPutRaceWithTrackedWalSync) { // Simulate full loss of unsynced data. This drops "key2" -> "val2" from the // DB WAL. - fault_env->DropUnsyncedFileData(); + ASSERT_OK(fault_env->DropUnsyncedFileData()); Reopen(options); @@ -536,7 +536,7 @@ TEST_P(DBWriteTest, InactiveWalFullySyncedBeforeUntracked) { // Simulate full loss of unsynced data. This should drop nothing since we did // `FlushWAL(true /* sync */)` before `Close()`. - fault_env->DropUnsyncedFileData(); + ASSERT_OK(fault_env->DropUnsyncedFileData()); Reopen(options); diff --git a/db/dbformat.cc b/db/dbformat.cc index 2d24c8953..63bb354de 100644 --- a/db/dbformat.cc +++ b/db/dbformat.cc @@ -88,6 +88,13 @@ void AppendKeyWithMaxTimestamp(std::string* result, const Slice& key, result->append(kTsMax.data(), ts_sz); } +void AppendUserKeyWithMinTimestamp(std::string* result, const Slice& key, + size_t ts_sz) { + assert(ts_sz > 0); + result->append(key.data(), key.size() - ts_sz); + result->append(ts_sz, static_cast(0)); +} + void AppendUserKeyWithMaxTimestamp(std::string* result, const Slice& key, size_t ts_sz) { assert(ts_sz > 0); diff --git a/db/dbformat.h b/db/dbformat.h index d0b3bbaaa..981866c09 100644 --- a/db/dbformat.h +++ b/db/dbformat.h @@ -168,10 +168,18 @@ inline void UnPackSequenceAndType(uint64_t packed, uint64_t* seq, EntryType GetEntryType(ValueType value_type); // Append the serialization of "key" to *result. +// +// input [internal key]: +// output before: empty +// output: void AppendInternalKey(std::string* result, const ParsedInternalKey& key); // Append the serialization of "key" to *result, replacing the original // timestamp with argument ts. +// +// input [internal key]: +// output before: empty +// output after: void AppendInternalKeyWithDifferentTimestamp(std::string* result, const ParsedInternalKey& key, const Slice& ts); @@ -179,37 +187,73 @@ void AppendInternalKeyWithDifferentTimestamp(std::string* result, // Serialized internal key consists of user key followed by footer. // This function appends the footer to *result, assuming that *result already // contains the user key at the end. +// +// output before: +// output after: void AppendInternalKeyFooter(std::string* result, SequenceNumber s, ValueType t); // Append the key and a minimal timestamp to *result +// +// input [user key without ts]: +// output before: empty +// output after: void AppendKeyWithMinTimestamp(std::string* result, const Slice& key, size_t ts_sz); // Append the key and a maximal timestamp to *result +// +// input [user key without ts]: +// output before: empty +// output after: void AppendKeyWithMaxTimestamp(std::string* result, const Slice& key, size_t ts_sz); +// `key` is a user key with timestamp. Append the user key without timestamp +// and the minimum timestamp to *result. +// +// input [user key]: +// output before: empty +// output after: +void AppendUserKeyWithMinTimestamp(std::string* result, const Slice& key, + size_t ts_sz); + // `key` is a user key with timestamp. Append the user key without timestamp // and the maximal timestamp to *result. +// +// input [user key]: +// output before: empty +// output after: void AppendUserKeyWithMaxTimestamp(std::string* result, const Slice& key, size_t ts_sz); // `key` is an internal key containing a user key without timestamp. Create a // new key in *result by padding a min timestamp of size `ts_sz` to the user key // and copying the remaining internal key bytes. +// +// input [internal key]: +// output before: empty +// output after: void PadInternalKeyWithMinTimestamp(std::string* result, const Slice& key, size_t ts_sz); // `key` is an internal key containing a user key with timestamp of size // `ts_sz`. Create a new internal key in *result by stripping the timestamp from // the user key and copying the remaining internal key bytes. +// +// input [internal key]: +// output before: empty +// output after: void StripTimestampFromInternalKey(std::string* result, const Slice& key, size_t ts_sz); // `key` is an internal key containing a user key with timestamp of size // `ts_sz`. Create a new internal key in *result while replace the original // timestamp with min timestamp. +// +// input [internal key]: +// output before: empty +// output after: void ReplaceInternalKeyWithMinTimestamp(std::string* result, const Slice& key, size_t ts_sz); @@ -221,11 +265,16 @@ Status ParseInternalKey(const Slice& internal_key, ParsedInternalKey* result, bool log_err_key); // Returns the user key portion of an internal key. +// +// input [internal key]: +// output: inline Slice ExtractUserKey(const Slice& internal_key) { assert(internal_key.size() >= kNumInternalBytes); return Slice(internal_key.data(), internal_key.size() - kNumInternalBytes); } +// input [internal key]: +// output : inline Slice ExtractUserKeyAndStripTimestamp(const Slice& internal_key, size_t ts_sz) { Slice ret = internal_key; @@ -233,17 +282,23 @@ inline Slice ExtractUserKeyAndStripTimestamp(const Slice& internal_key, return ret; } +// input [user key]: +// output: inline Slice StripTimestampFromUserKey(const Slice& user_key, size_t ts_sz) { Slice ret = user_key; ret.remove_suffix(ts_sz); return ret; } +// input [user key]: +// output: inline Slice ExtractTimestampFromUserKey(const Slice& user_key, size_t ts_sz) { assert(user_key.size() >= ts_sz); return Slice(user_key.data() + user_key.size() - ts_sz, ts_sz); } +// input [internal key]: +// output: inline Slice ExtractTimestampFromKey(const Slice& internal_key, size_t ts_sz) { const size_t key_size = internal_key.size(); assert(key_size >= kNumInternalBytes + ts_sz); @@ -251,12 +306,16 @@ inline Slice ExtractTimestampFromKey(const Slice& internal_key, size_t ts_sz) { ts_sz); } +// input [internal key]: +// output: inline uint64_t ExtractInternalKeyFooter(const Slice& internal_key) { assert(internal_key.size() >= kNumInternalBytes); const size_t n = internal_key.size(); return DecodeFixed64(internal_key.data() + n - kNumInternalBytes); } +// input [internal key]: +// output: inline ValueType ExtractValueType(const Slice& internal_key) { uint64_t num = ExtractInternalKeyFooter(internal_key); unsigned char c = num & 0xff; @@ -296,6 +355,7 @@ class InternalKeyComparator // Same as Compare except that it excludes the value type from comparison int CompareKeySeq(const Slice& a, const Slice& b) const; + int CompareKeySeq(const ParsedInternalKey& a, const Slice& b) const; const Comparator* user_comparator() const { return user_comparator_.user_comparator(); @@ -917,6 +977,26 @@ inline int InternalKeyComparator::CompareKeySeq(const Slice& akey, return r; } +inline int InternalKeyComparator::CompareKeySeq(const ParsedInternalKey& a, + const Slice& b) const { + // Order by: + // increasing user key (according to user-supplied comparator) + // decreasing sequence number + int r = user_comparator_.Compare(a.user_key, ExtractUserKey(b)); + if (r == 0) { + // Shift the number to exclude the last byte which contains the value type + const uint64_t anum = a.sequence; + const uint64_t bnum = + DecodeFixed64(b.data() + b.size() - kNumInternalBytes) >> 8; + if (anum > bnum) { + r = -1; + } else if (anum < bnum) { + r = +1; + } + } + return r; +} + inline int InternalKeyComparator::Compare(const Slice& a, SequenceNumber a_global_seqno, const Slice& b, diff --git a/db/deletefile_test.cc b/db/deletefile_test.cc index 481eda7dd..b6d4f559e 100644 --- a/db/deletefile_test.cc +++ b/db/deletefile_test.cc @@ -575,6 +575,7 @@ TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) { ASSERT_OK(itr->status()); ++count; } + ASSERT_OK(itr->status()); ASSERT_EQ(count, 1000); } @@ -588,6 +589,7 @@ TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) { ASSERT_OK(itr->status()); ++count; } + ASSERT_OK(itr->status()); ASSERT_EQ(count, 1000); } } diff --git a/db/error_handler.cc b/db/error_handler.cc index 55821952d..f43261001 100644 --- a/db/error_handler.cc +++ b/db/error_handler.cc @@ -320,7 +320,7 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err, // Check if recovery is currently in progress. If it is, we will save this // error so we can check it at the end to see if recovery succeeded or not if (recovery_in_prog_ && recovery_error_.ok()) { - recovery_error_ = new_bg_err; + recovery_error_ = status_to_io_status(Status(new_bg_err)); } bool auto_recovery = auto_recovery_; @@ -396,16 +396,6 @@ const Status& ErrorHandler::SetBGError(const Status& bg_status, ROCKS_LOG_WARN(db_options_.info_log, "Background IO error %s", bg_io_err.ToString().c_str()); - if (recovery_in_prog_ && recovery_io_error_.ok()) { - recovery_io_error_ = bg_io_err; - } - if (BackgroundErrorReason::kManifestWrite == reason || - BackgroundErrorReason::kManifestWriteNoWAL == reason) { - // Always returns ok - ROCKS_LOG_INFO(db_options_.info_log, "Disabling File Deletions"); - db_->DisableFileDeletionsWithLock().PermitUncheckedError(); - } - Status new_bg_io_err = bg_io_err; DBRecoverContext context; if (bg_io_err.GetScope() != IOStatus::IOErrorScope::kIOErrorScopeFile && @@ -472,6 +462,8 @@ const Status& ErrorHandler::SetBGError(const Status& bg_status, ROCKS_LOG_INFO( db_options_.info_log, "ErrorHandler: Compaction will schedule by itself to resume\n"); + // Not used in this code path. + new_bg_io_err.PermitUncheckedError(); return bg_error_; } else if (BackgroundErrorReason::kFlushNoWAL == reason || BackgroundErrorReason::kManifestWriteNoWAL == reason) { @@ -500,14 +492,35 @@ const Status& ErrorHandler::SetBGError(const Status& bg_status, RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_IO_ERROR_COUNT_MISSPELLED); } - // HandleKnownErrors() will use recovery_error_, so ignore - // recovery_io_error_. - // TODO: Do some refactoring and use only one recovery_error_ - recovery_io_error_.PermitUncheckedError(); return HandleKnownErrors(new_bg_io_err, reason); } } +void ErrorHandler::AddFilesToQuarantine( + autovector*> files_to_quarantine) { + db_mutex_->AssertHeld(); + std::ostringstream quarantine_files_oss; + bool is_first_one = true; + for (const auto* files : files_to_quarantine) { + assert(files); + for (uint64_t file_number : *files) { + files_to_quarantine_.push_back(file_number); + quarantine_files_oss << (is_first_one ? "" : ", ") << file_number; + is_first_one = false; + } + } + ROCKS_LOG_INFO(db_options_.info_log, + "ErrorHandler: added file numbers %s to quarantine.\n", + quarantine_files_oss.str().c_str()); +} + +void ErrorHandler::ClearFilesToQuarantine() { + db_mutex_->AssertHeld(); + files_to_quarantine_.clear(); + ROCKS_LOG_INFO(db_options_.info_log, + "ErrorHandler: cleared files in quarantine.\n"); +} + Status ErrorHandler::OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery) { if (bg_error.severity() >= Status::Severity::kFatalError) { @@ -555,14 +568,16 @@ Status ErrorHandler::ClearBGError() { // Signal that recovery succeeded if (recovery_error_.ok()) { + assert(files_to_quarantine_.empty()); Status old_bg_error = bg_error_; // old_bg_error is only for notifying listeners, so may not be checked old_bg_error.PermitUncheckedError(); // Clear and check the recovery IO and BG error + is_db_stopped_.store(false, std::memory_order_release); bg_error_ = Status::OK(); - recovery_io_error_ = IOStatus::OK(); + recovery_error_ = IOStatus::OK(); bg_error_.PermitUncheckedError(); - recovery_io_error_.PermitUncheckedError(); + recovery_error_.PermitUncheckedError(); recovery_in_prog_ = false; soft_error_no_bg_work_ = false; EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, old_bg_error, @@ -600,14 +615,14 @@ Status ErrorHandler::RecoverFromBGError(bool is_manual) { if (bg_error_.severity() == Status::Severity::kSoftError && recover_context_.flush_reason == FlushReason::kErrorRecovery) { // Simply clear the background error and return - recovery_error_ = Status::OK(); + recovery_error_ = IOStatus::OK(); return ClearBGError(); } // Reset recovery_error_. We will use this to record any errors that happen // during the recovery process. While recovering, the only operations that // can generate background errors should be the flush operations - recovery_error_ = Status::OK(); + recovery_error_ = IOStatus::OK(); recovery_error_.PermitUncheckedError(); Status s = db_->ResumeImpl(recover_context_); if (s.ok()) { @@ -636,6 +651,13 @@ const Status& ErrorHandler::StartRecoverFromRetryableBGIOError( } else if (db_options_.max_bgerror_resume_count <= 0 || recovery_in_prog_) { // Auto resume BG error is not enabled, directly return bg_error_. return bg_error_; + } else if (end_recovery_) { + // Can temporarily release db mutex + EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, + Status::ShutdownInProgress(), + db_mutex_); + db_mutex_->AssertHeld(); + return bg_error_; } if (bg_error_stats_ != nullptr) { RecordTick(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_COUNT); @@ -643,20 +665,30 @@ const Status& ErrorHandler::StartRecoverFromRetryableBGIOError( ROCKS_LOG_INFO( db_options_.info_log, "ErrorHandler: Call StartRecoverFromRetryableBGIOError to resume\n"); + // Needs to be set in the same lock hold as setting BG error, otherwise + // intervening writes could see a BG error without a recovery and bail out. + recovery_in_prog_ = true; + if (recovery_thread_) { + // Ensure only one thread can execute the join(). + std::unique_ptr old_recovery_thread( + std::move(recovery_thread_)); // In this case, if recovery_in_prog_ is false, current thread should // wait the previous recover thread to finish and create a new thread // to recover from the bg error. db_mutex_->Unlock(); - recovery_thread_->join(); + TEST_SYNC_POINT( + "StartRecoverFromRetryableBGIOError:BeforeWaitingForOtherThread"); + old_recovery_thread->join(); + TEST_SYNC_POINT( + "StartRecoverFromRetryableBGIOError:AfterWaitingForOtherThread"); db_mutex_->Lock(); } - recovery_in_prog_ = true; recovery_thread_.reset( new port::Thread(&ErrorHandler::RecoverFromRetryableBGIOError, this)); - if (recovery_io_error_.ok() && recovery_error_.ok()) { + if (recovery_error_.ok()) { return recovery_error_; } else { return bg_error_; @@ -666,15 +698,20 @@ const Status& ErrorHandler::StartRecoverFromRetryableBGIOError( // Automatic recover from Retryable BG IO error. Must be called after db // mutex is released. void ErrorHandler::RecoverFromRetryableBGIOError() { + assert(recovery_in_prog_); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart"); + TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart2"); InstrumentedMutexLock l(db_mutex_); if (end_recovery_) { EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, Status::ShutdownInProgress(), db_mutex_); + + recovery_in_prog_ = false; return; } DBRecoverContext context = recover_context_; + context.flush_after_recovery = true; int resume_count = db_options_.max_bgerror_resume_count; uint64_t wait_interval = db_options_.bgerror_resume_retry_interval; uint64_t retry_count = 0; @@ -684,12 +721,12 @@ void ErrorHandler::RecoverFromRetryableBGIOError() { EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, Status::ShutdownInProgress(), db_mutex_); + recovery_in_prog_ = false; return; } TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeResume0"); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeResume1"); - recovery_io_error_ = IOStatus::OK(); - recovery_error_ = Status::OK(); + recovery_error_ = IOStatus::OK(); retry_count++; Status s = db_->ResumeImpl(context); if (bg_error_stats_ != nullptr) { @@ -709,9 +746,9 @@ void ErrorHandler::RecoverFromRetryableBGIOError() { bg_error_, db_mutex_); return; } - if (!recovery_io_error_.ok() && + if (!recovery_error_.ok() && recovery_error_.severity() <= Status::Severity::kHardError && - recovery_io_error_.GetRetryable()) { + recovery_error_.GetRetryable()) { // If new BG IO error happens during auto recovery and it is retryable // and its severity is Hard Error or lower, the auto resmue sleep for // a period of time and redo auto resume if it is allowed. @@ -720,33 +757,23 @@ void ErrorHandler::RecoverFromRetryableBGIOError() { int64_t wait_until = db_options_.clock->NowMicros() + wait_interval; cv_.TimedWait(wait_until); } else { - // There are three possibility: 1) recover_io_error is set during resume + // There are three possibility: 1) recovery_error_ is set during resume // and the error is not retryable, 2) recover is successful, 3) other // error happens during resume and cannot be resumed here. - if (recovery_io_error_.ok() && recovery_error_.ok() && s.ok()) { + if (recovery_error_.ok() && s.ok()) { // recover from the retryable IO error and no other BG errors. Clean // the bg_error and notify user. TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverSuccess"); - Status old_bg_error = bg_error_; - is_db_stopped_.store(false, std::memory_order_release); - bg_error_ = Status::OK(); - bg_error_.PermitUncheckedError(); - EventHelpers::NotifyOnErrorRecoveryEnd( - db_options_.listeners, old_bg_error, bg_error_, db_mutex_); if (bg_error_stats_ != nullptr) { RecordTick(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT); RecordInHistogram(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count); } - recovery_in_prog_ = false; - if (soft_error_no_bg_work_) { - soft_error_no_bg_work_ = false; - } return; } else { - // In this case: 1) recovery_io_error is more serious or not retryable - // 2) other Non IO recovery_error happens. The auto recovery stops. + // In this case: 1) recovery_error_ is more serious or not retryable + // 2) other error happens. The auto recovery stops. recovery_in_prog_ = false; if (bg_error_stats_ != nullptr) { RecordInHistogram(bg_error_stats_.get(), @@ -754,10 +781,7 @@ void ErrorHandler::RecoverFromRetryableBGIOError() { } EventHelpers::NotifyOnErrorRecoveryEnd( db_options_.listeners, bg_error_, - !recovery_io_error_.ok() - ? recovery_io_error_ - : (!recovery_error_.ok() ? recovery_error_ : s), - db_mutex_); + !recovery_error_.ok() ? recovery_error_ : s, db_mutex_); return; } } @@ -777,7 +801,7 @@ void ErrorHandler::RecoverFromRetryableBGIOError() { void ErrorHandler::CheckAndSetRecoveryAndBGError(const Status& bg_err) { if (recovery_in_prog_ && recovery_error_.ok()) { - recovery_error_ = bg_err; + recovery_error_ = status_to_io_status(Status(bg_err)); } if (bg_err.severity() > bg_error_.severity()) { bg_error_ = bg_err; @@ -793,12 +817,16 @@ void ErrorHandler::EndAutoRecovery() { if (!end_recovery_) { end_recovery_ = true; } - cv_.SignalAll(); - db_mutex_->Unlock(); if (recovery_thread_) { - recovery_thread_->join(); + // Ensure only one thread can execute the join(). + std::unique_ptr old_recovery_thread( + std::move(recovery_thread_)); + db_mutex_->Unlock(); + cv_.SignalAll(); + old_recovery_thread->join(); + db_mutex_->Lock(); } - db_mutex_->Lock(); + TEST_SYNC_POINT("PostEndAutoRecovery"); return; } diff --git a/db/error_handler.h b/db/error_handler.h index 34e08a525..1168d91fa 100644 --- a/db/error_handler.h +++ b/db/error_handler.h @@ -4,11 +4,14 @@ // (found in the LICENSE.Apache file in the root directory). #pragma once +#include + #include "monitoring/instrumented_mutex.h" #include "options/db_options.h" #include "rocksdb/io_status.h" #include "rocksdb/listener.h" #include "rocksdb/status.h" +#include "util/autovector.h" namespace ROCKSDB_NAMESPACE { @@ -19,10 +22,13 @@ class DBImpl; // FlushReason, which tells the flush job why this flush is called. struct DBRecoverContext { FlushReason flush_reason; + bool flush_after_recovery; - DBRecoverContext() : flush_reason(FlushReason::kErrorRecovery) {} - - DBRecoverContext(FlushReason reason) : flush_reason(reason) {} + DBRecoverContext() + : flush_reason(FlushReason::kErrorRecovery), + flush_after_recovery(false) {} + DBRecoverContext(FlushReason reason) + : flush_reason(reason), flush_after_recovery(false) {} }; class ErrorHandler { @@ -43,7 +49,6 @@ class ErrorHandler { // Clear the checked flag for uninitialized errors bg_error_.PermitUncheckedError(); recovery_error_.PermitUncheckedError(); - recovery_io_error_.PermitUncheckedError(); } void EnableAutoRecovery() { auto_recovery_ = true; } @@ -78,16 +83,23 @@ class ErrorHandler { void EndAutoRecovery(); + void AddFilesToQuarantine( + autovector*> files_to_quarantine); + + const autovector& GetFilesToQuarantine() const { + db_mutex_->AssertHeld(); + return files_to_quarantine_; + } + + void ClearFilesToQuarantine(); + private: DBImpl* db_; const ImmutableDBOptions& db_options_; Status bg_error_; // A separate Status variable used to record any errors during the // recovery process from hard errors - Status recovery_error_; - // A separate IO Status variable used to record any IO errors during - // the recovery process. At the same time, recovery_error_ is also set. - IOStatus recovery_io_error_; + IOStatus recovery_error_; // The condition variable used with db_mutex during auto resume for time // wait. InstrumentedCondVar cv_; @@ -109,6 +121,13 @@ class ErrorHandler { // The pointer of DB statistics. std::shared_ptr bg_error_stats_; + // During recovery from manifest IO errors, files whose VersionEdits entries + // could be in an ambiguous state are quarantined and file deletion refrain + // from deleting them. Successful recovery will clear this vector. Files are + // added to this vector while DB mutex was locked, this data structure is + // unsorted. + autovector files_to_quarantine_; + const Status& HandleKnownErrors(const Status& bg_err, BackgroundErrorReason reason); Status OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery); diff --git a/db/error_handler_fs_test.cc b/db/error_handler_fs_test.cc index 442631ded..2d33a7a69 100644 --- a/db/error_handler_fs_test.cc +++ b/db/error_handler_fs_test.cc @@ -7,12 +7,15 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include + #include "db/db_test_util.h" #include "file/sst_file_manager_impl.h" #include "port/stack_trace.h" #include "rocksdb/io_status.h" #include "rocksdb/sst_file_manager.h" #include "test_util/sync_point.h" +#include "test_util/testharness.h" #include "util/random.h" #include "utilities/fault_injection_env.h" #include "utilities/fault_injection_fs.h" @@ -152,9 +155,9 @@ class ErrorHandlerFSListener : public EventListener { FaultInjectionTestFS* fault_fs_; }; -TEST_F(DBErrorHandlingFSTest, FLushWriteError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -197,9 +200,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteError) { // All the NoSpace IOError will be handled as the regular BG Error no matter the // retryable flag is set of not. So the auto resume for retryable IO Error will // not be triggered. Also, it is mapped as hard error. -TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteNoSpaceError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -241,9 +244,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteRetryableError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -315,9 +318,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWriteFileScopeError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteFileScopeError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -399,9 +402,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteFileScopeError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWALWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWALWriteRetryableError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -443,9 +446,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWALWriteRetryableError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWALAtomicWriteRetryableError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -489,9 +492,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) { } // The flush error is injected before we finish the table build -TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableError1) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -545,9 +548,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { } // The retryable IO error is injected before we sync table -TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteNoWALRetryableError2) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -589,9 +592,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) { } // The retryable IO error is injected before we close the table file -TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteNoWALRetryableError3) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -633,8 +636,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -658,6 +661,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty()); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); fault_fs_->SetFilesystemActive(true); @@ -666,6 +670,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { new_manifest = GetManifestNameFromLiveFiles(); ASSERT_NE(new_manifest, old_manifest); + ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty()); Reopen(options); ASSERT_EQ("val", Get(Key(0))); @@ -674,8 +679,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -701,6 +706,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty()); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); fault_fs_->SetFilesystemActive(true); @@ -709,6 +715,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { new_manifest = GetManifestNameFromLiveFiles(); ASSERT_NE(new_manifest, old_manifest); + ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty()); Reopen(options); ASSERT_EQ("val", Get(Key(0))); @@ -717,8 +724,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -746,6 +753,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) { [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); + ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty()); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); @@ -755,6 +763,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) { new_manifest = GetManifestNameFromLiveFiles(); ASSERT_NE(new_manifest, old_manifest); + ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty()); Reopen(options); ASSERT_EQ("val", Get(Key(0))); @@ -763,8 +772,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -792,6 +801,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) { SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty()); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); fault_fs_->SetFilesystemActive(true); @@ -800,6 +810,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) { new_manifest = GetManifestNameFromLiveFiles(); ASSERT_NE(new_manifest, old_manifest); + ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty()); Reopen(options); ASSERT_EQ("val", Get(Key(0))); @@ -808,8 +819,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) { } TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -833,11 +844,13 @@ TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) { SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty()); fault_fs_->SetFilesystemActive(true); // This Resume() will attempt to create a new manifest file and fail again s = dbfull()->Resume(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty()); fault_fs_->SetFilesystemActive(true); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); @@ -848,6 +861,7 @@ TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) { new_manifest = GetManifestNameFromLiveFiles(); ASSERT_NE(new_manifest, old_manifest); + ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty()); Reopen(options); ASSERT_EQ("val", Get(Key(0))); @@ -860,8 +874,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); return; } - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -914,6 +928,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { fault_fs_->SetFilesystemActive(true); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); TEST_SYNC_POINT("CompactionManifestWriteError:1"); + ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty()); TEST_SYNC_POINT("CompactionManifestWriteError:2"); s = dbfull()->TEST_WaitForCompact(); @@ -922,6 +937,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { new_manifest = GetManifestNameFromLiveFiles(); ASSERT_NE(new_manifest, old_manifest); + ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty()); Reopen(options); ASSERT_EQ("val", Get(Key(0))); ASSERT_EQ("val", Get(Key(1))); @@ -930,8 +946,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { } TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -980,6 +996,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { ASSERT_OK(s); TEST_SYNC_POINT("CompactionManifestWriteError:0"); + ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty()); TEST_SYNC_POINT("CompactionManifestWriteError:1"); s = dbfull()->TEST_WaitForCompact(); @@ -993,6 +1010,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { new_manifest = GetManifestNameFromLiveFiles(); ASSERT_NE(new_manifest, old_manifest); + ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty()); Reopen(options); ASSERT_EQ("val", Get(Key(0))); @@ -1002,8 +1020,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, CompactionWriteError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1044,8 +1062,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteError) { } TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1091,8 +1109,8 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteFileScopeError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1183,8 +1201,8 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) { ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); return; } - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1228,8 +1246,8 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) { } TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1257,8 +1275,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) { ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); return; } - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1330,8 +1348,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) { } TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1425,8 +1443,8 @@ TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) { ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); return; } - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1757,10 +1775,10 @@ TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) { // to soft error and trigger auto resume. During auto resume, SwitchMemtable // is disabled to avoid small SST tables. Write can still be applied before // the bg error is cleaned unless the memtable is full. -TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { +TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableErrorAutoRecover1) { // Activate the FS before the first resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1781,7 +1799,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { ASSERT_OK(Put(Key(1), "val1", wo)); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( {{"RecoverFromRetryableBGIOError:LoopOut", - "FLushWritNoWALRetryableeErrorAutoRecover1:1"}}); + "FlushWritNoWALRetryableeErrorAutoRecover1:1"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); @@ -1790,7 +1808,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { s = Flush(); ASSERT_EQ("val1", Get(Key(1))); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); - TEST_SYNC_POINT("FLushWritNoWALRetryableeErrorAutoRecover1:1"); + TEST_SYNC_POINT("FlushWritNoWALRetryableeErrorAutoRecover1:1"); ASSERT_EQ("val1", Get(Key(1))); ASSERT_EQ("val1", Get(Key(1))); SyncPoint::GetInstance()->DisableProcessing(); @@ -1827,10 +1845,94 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) { +TEST_F(DBErrorHandlingFSTest, MultipleRecoveryThreads) { + // This test creates a scenario where second write's recovery can get started + // while mutex is released for a short period during + // NotifyOnErrorRecoveryEnd() from the first write's recovery. This is to make + // sure RecoverFromRetryableBGIOError() from the second write's recovery + // thread does not start with recovery_in_prog_ = false; + + std::shared_ptr listener = + std::make_shared(); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 100; + options.bgerror_resume_retry_interval = 1000000; // 1 second + options.statistics = CreateDBStatistics(); + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + fault_fs_->SetFilesystemActive(false, error_msg); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"NotifyOnErrorRecoveryEnd:MutexUnlocked:1", + "MultipleRecoveryThreads:1"}, + {"MultipleRecoveryThreads:2", + "NotifyOnErrorRecoveryEnd:MutexUnlocked:2"}, + {"StartRecoverFromRetryableBGIOError:BeforeWaitingForOtherThread", + "MultipleRecoveryThreads:3"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "MultipleRecoveryThreads:4"}, + {"MultipleRecoveryThreads:4", + "StartRecoverFromRetryableBGIOError:AfterWaitingForOtherThread"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + // First write with read fault injected and recovery will start + { + ASSERT_OK(Put(Key(1), "val1", wo)); + Status s = Flush(); + ASSERT_NOK(s); + } + // Remove read fault injection so that first recovery can go through + fault_fs_->SetFilesystemActive(true); + + // At this point, first recovery is now at NotifyOnErrorRecoveryEnd. Mutex is + // released. + TEST_SYNC_POINT("MultipleRecoveryThreads:1"); + + ROCKSDB_NAMESPACE::port::Thread second_write([&] { + // Second write with read fault injected + fault_fs_->SetFilesystemActive(false, error_msg); + ASSERT_OK(Put(Key(2), "val2", wo)); + Status s = Flush(); + ASSERT_NOK(s); + }); + // Second bg thread before waiting for the first thread's recovery thread + TEST_SYNC_POINT("MultipleRecoveryThreads:3"); + // First thread's recovery thread continues + TEST_SYNC_POINT("MultipleRecoveryThreads:2"); + // Wait for the first thread's recovery to finish + // (this sets recovery_in_prog_ = false) + // And second thread continues and starts recovery thread + TEST_SYNC_POINT("MultipleRecoveryThreads:4"); + second_write.join(); + // Remove error injection so that second thread recovery can go through + fault_fs_->SetFilesystemActive(true); + + // Set up sync point so that we can wait for the recovery thread to finish + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"RecoverFromRetryableBGIOError:RecoverSuccess", + "MultipleRecoveryThreads:6"}}); + + // Wait for the second thread's recovery to be done + TEST_SYNC_POINT("MultipleRecoveryThreads:6"); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableErrorAutoRecover2) { // Activate the FS before the first resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1888,10 +1990,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) { // Auto resume fromt the flush retryable IO error. Activate the FS before the // first resume. Resume is successful -TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) { +TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAutoRecover1) { // Activate the FS before the first resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1930,10 +2032,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) { // Auto resume fromt the flush retryable IO error and set the retry limit count. // Never activate the FS and auto resume should fail at the end -TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) { +TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAutoRecover2) { // Fail all the resume and let user to resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1950,18 +2052,18 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) { ASSERT_OK(Put(Key(1), "val1")); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( - {{"FLushWritRetryableeErrorAutoRecover2:0", + {{"FlushWritRetryableeErrorAutoRecover2:0", "RecoverFromRetryableBGIOError:BeforeStart"}, {"RecoverFromRetryableBGIOError:LoopOut", - "FLushWritRetryableeErrorAutoRecover2:1"}}); + "FlushWritRetryableeErrorAutoRecover2:1"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); - TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:0"); - TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:1"); + TEST_SYNC_POINT("FlushWritRetryableeErrorAutoRecover2:0"); + TEST_SYNC_POINT("FlushWritRetryableeErrorAutoRecover2:1"); fault_fs_->SetFilesystemActive(true); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); @@ -1983,8 +2085,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) { // Fail the first resume and let the second resume be successful. TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) { // Fail the first resume and let the second resume be successful - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2036,8 +2138,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) { TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableErrorAutoRecover) { // Fail the first resume and let the second resume be successful - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2091,8 +2193,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableErrorAutoRecover) { TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableErrorAutoRecover) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2181,8 +2283,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) { // compaction, the FS is set to active and compaction is successful, so // the test will hit the CompactionJob::FinishCompactionOutputFile1 sync // point. - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2235,8 +2337,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) { } TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2338,8 +2440,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) { // Fail the first recover and try second time. - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2439,10 +2541,10 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) { // Fail auto resume from a flush retryable error and verify that // OnErrorRecoveryEnd listener callback is called -TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAbortRecovery) { +TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAbortRecovery) { // Activate the FS before the first resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2473,6 +2575,59 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAbortRecovery) { Destroy(options); } +TEST_F(DBErrorHandlingFSTest, FlushErrorRecoveryRaceWithDBDestruction) { + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + std::shared_ptr listener = + std::make_shared(); + options.listeners.emplace_back(listener); + DestroyAndReopen(options); + ASSERT_OK(Put("k1", "val")); + + // Inject retryable flush error + bool error_set = false; + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeOutputValidation", [&](void*) { + if (error_set) { + return; + } + IOStatus st = IOStatus::IOError("Injected"); + st.SetRetryable(true); + fault_fs_->SetFilesystemActive(false, st); + error_set = true; + }); + + port::Thread db_close_thread; + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeDeleteFile", [&](void*) { + // Clear retryable flush error injection + fault_fs_->SetFilesystemActive(true); + + // Coerce race between ending auto recovery in db destruction and flush + // error recovery + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"PostEndAutoRecovery", "FlushJob::WriteLevel0Table"}}); + db_close_thread = port::Thread([&] { Close(); }); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = Flush(); + ASSERT_NOK(s); + + int placeholder = 1; + listener->WaitForRecovery(placeholder); + ASSERT_TRUE(listener->new_bg_error().IsShutdownInProgress()); + + // Prior to the fix, the db close will crash due to the recovery thread for + // flush error is not joined by the time of destruction. + db_close_thread.join(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Destroy(options); +} + TEST_F(DBErrorHandlingFSTest, FlushReadError) { std::shared_ptr listener = std::make_shared(); @@ -2565,8 +2720,8 @@ TEST_F(DBErrorHandlingFSTest, AtomicFlushReadError) { s = dbfull()->TEST_GetBGError(); ASSERT_OK(s); - TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, - GetDefaultOptions()); + ASSERT_OK(TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, + GetDefaultOptions())); ASSERT_EQ("val", Get(Key(0))); } @@ -2606,8 +2761,8 @@ TEST_F(DBErrorHandlingFSTest, AtomicFlushNoSpaceError) { s = dbfull()->TEST_GetBGError(); ASSERT_OK(s); - TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, - GetDefaultOptions()); + ASSERT_OK(TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, + GetDefaultOptions())); ASSERT_EQ("val", Get(Key(0))); } @@ -2680,9 +2835,9 @@ TEST_F(DBErrorHandlingFSTest, CompactionReadRetryableErrorAutoRecover) { class DBErrorHandlingFencingTest : public DBErrorHandlingFSTest, public testing::WithParamInterface {}; -TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_P(DBErrorHandlingFencingTest, FlushWriteFenced) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2709,8 +2864,8 @@ TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) { } TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2744,8 +2899,8 @@ TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) { } TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2785,8 +2940,8 @@ TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) { } TEST_P(DBErrorHandlingFencingTest, WALWriteFenced) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; diff --git a/db/event_helpers.cc b/db/event_helpers.cc index 4360144ec..65f6a5a48 100644 --- a/db/event_helpers.cc +++ b/db/event_helpers.cc @@ -122,6 +122,8 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( << "column_family_name" << table_properties.column_family_name << "column_family_id" << table_properties.column_family_id << "comparator" << table_properties.comparator_name + << "user_defined_timestamps_persisted" + << table_properties.user_defined_timestamps_persisted << "merge_operator" << table_properties.merge_operator_name << "prefix_extractor_name" << table_properties.prefix_extractor_name << "property_collectors" @@ -228,6 +230,8 @@ void EventHelpers::NotifyOnErrorRecoveryEnd( db_mutex->AssertHeld(); // release lock while notifying events db_mutex->Unlock(); + TEST_SYNC_POINT("NotifyOnErrorRecoveryEnd:MutexUnlocked:1"); + TEST_SYNC_POINT("NotifyOnErrorRecoveryEnd:MutexUnlocked:2"); for (auto& listener : listeners) { BackgroundErrorRecoveryInfo info; info.old_bg_error = old_bg_error; @@ -238,6 +242,8 @@ void EventHelpers::NotifyOnErrorRecoveryEnd( info.new_bg_error.PermitUncheckedError(); } db_mutex->Lock(); + } else { + old_bg_error.PermitUncheckedError(); } } diff --git a/db/external_sst_file_basic_test.cc b/db/external_sst_file_basic_test.cc index 39334994a..41ac48a07 100644 --- a/db/external_sst_file_basic_test.cc +++ b/db/external_sst_file_basic_test.cc @@ -9,6 +9,8 @@ #include "db/version_edit.h" #include "port/port.h" #include "port/stack_trace.h" +#include "rocksdb/advanced_options.h" +#include "rocksdb/options.h" #include "rocksdb/sst_file_writer.h" #include "test_util/testharness.h" #include "test_util/testutil.h" @@ -1292,6 +1294,80 @@ TEST_F(ExternalSSTFileBasicTest, VerifyChecksumReadahead) { Destroy(options); } +TEST_F(ExternalSSTFileBasicTest, ReadOldValueOfIngestedKeyBug) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.disable_auto_compactions = true; + options.num_levels = 3; + options.preserve_internal_time_seconds = 36000; + DestroyAndReopen(options); + + // To create the following LSM tree to trigger the bug: + // L0 + // L1 with seqno [1, 2] + // L2 with seqno [3, 4] + + // To create L1 shape + ASSERT_OK( + db_->Put(WriteOptions(), db_->DefaultColumnFamily(), "k1", "seqno1")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK( + db_->Put(WriteOptions(), db_->DefaultColumnFamily(), "k1", "seqno2")); + ASSERT_OK(db_->Flush(FlushOptions())); + ColumnFamilyMetaData meta_1; + db_->GetColumnFamilyMetaData(&meta_1); + auto& files_1 = meta_1.levels[0].files; + ASSERT_EQ(files_1.size(), 2); + std::string file1 = files_1[0].db_path + files_1[0].name; + std::string file2 = files_1[1].db_path + files_1[1].name; + ASSERT_OK(db_->CompactFiles(CompactionOptions(), {file1, file2}, 1)); + // To confirm L1 shape + ColumnFamilyMetaData meta_2; + db_->GetColumnFamilyMetaData(&meta_2); + ASSERT_EQ(meta_2.levels[0].files.size(), 0); + ASSERT_EQ(meta_2.levels[1].files.size(), 1); + // Seqno starts from non-zero due to seqno reservation for + // preserve_internal_time_seconds greater than 0; + ASSERT_EQ(meta_2.levels[1].files[0].largest_seqno, 102); + ASSERT_EQ(meta_2.levels[2].files.size(), 0); + // To create L2 shape + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), "k2overlap", + "old_value")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), "k2overlap", + "old_value")); + ASSERT_OK(db_->Flush(FlushOptions())); + ColumnFamilyMetaData meta_3; + db_->GetColumnFamilyMetaData(&meta_3); + auto& files_3 = meta_3.levels[0].files; + std::string file3 = files_3[0].db_path + files_3[0].name; + std::string file4 = files_3[1].db_path + files_3[1].name; + ASSERT_OK(db_->CompactFiles(CompactionOptions(), {file3, file4}, 2)); + // To confirm L2 shape + ColumnFamilyMetaData meta_4; + db_->GetColumnFamilyMetaData(&meta_4); + ASSERT_EQ(meta_4.levels[0].files.size(), 0); + ASSERT_EQ(meta_4.levels[1].files.size(), 1); + ASSERT_EQ(meta_4.levels[2].files.size(), 1); + ASSERT_EQ(meta_4.levels[2].files[0].largest_seqno, 104); + + // Ingest a file with new value of the key "k2overlap" + SstFileWriter sst_file_writer(EnvOptions(), options); + std::string f = sst_files_dir_ + "f.sst"; + ASSERT_OK(sst_file_writer.Open(f)); + ASSERT_OK(sst_file_writer.Put("k2overlap", "new_value")); + ExternalSstFileInfo f_info; + ASSERT_OK(sst_file_writer.Finish(&f_info)); + ASSERT_OK(db_->IngestExternalFile({f}, IngestExternalFileOptions())); + + // To verify new value of the key "k2overlap" is correctly returned + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), "k2overlap", &value)); + // Before the fix, the value would be "old_value" and assertion failed + ASSERT_EQ(value, "new_value"); +} + TEST_F(ExternalSSTFileBasicTest, IngestRangeDeletionTombstoneWithGlobalSeqno) { for (int i = 5; i < 25; i++) { ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(i), @@ -1346,7 +1422,7 @@ TEST_P(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) { // range del [0, 50) in L6 file, [50, 100) in L0 file, [100, 150) in memtable for (int i = 0; i < 3; i++) { if (i != 0) { - db_->Flush(FlushOptions()); + ASSERT_OK(db_->Flush(FlushOptions())); if (i == 1) { MoveFilesToLevel(kNumLevels - 1); } @@ -1548,6 +1624,11 @@ TEST_F(ExternalSSTFileBasicTest, RangeDeletionEndComesBeforeStart) { } TEST_P(ExternalSSTFileBasicTest, IngestFileWithBadBlockChecksum) { + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + if (!verify_checksums_before_ingest) { + ROCKSDB_GTEST_BYPASS("Bypassing test when !verify_checksums_before_ingest"); + return; + } bool change_checksum_called = false; const auto& change_checksum = [&](void* arg) { if (!change_checksum_called) { @@ -1565,24 +1646,20 @@ TEST_P(ExternalSSTFileBasicTest, IngestFileWithBadBlockChecksum) { SyncPoint::GetInstance()->EnableProcessing(); int file_id = 0; bool write_global_seqno = std::get<0>(GetParam()); - bool verify_checksums_before_ingest = std::get<1>(GetParam()); do { Options options = CurrentOptions(); DestroyAndReopen(options); std::map true_data; Status s = GenerateAndAddExternalFile( options, {1, 2, 3, 4, 5, 6}, ValueType::kTypeValue, file_id++, - write_global_seqno, verify_checksums_before_ingest, &true_data); - if (verify_checksums_before_ingest) { - ASSERT_NOK(s); - } else { - ASSERT_OK(s); - } + write_global_seqno, /*verify_checksums_before_ingest=*/true, + &true_data); + ASSERT_NOK(s); change_checksum_called = false; } while (ChangeOptionsForFileIngestionTest()); } -TEST_P(ExternalSSTFileBasicTest, IngestFileWithFirstByteTampered) { +TEST_P(ExternalSSTFileBasicTest, IngestFileWithCorruptedDataBlock) { if (!random_rwfile_supported_) { ROCKSDB_GTEST_SKIP("Test requires NewRandomRWFile support"); return; @@ -1590,15 +1667,21 @@ TEST_P(ExternalSSTFileBasicTest, IngestFileWithFirstByteTampered) { SyncPoint::GetInstance()->DisableProcessing(); int file_id = 0; EnvOptions env_options; + Random rnd(301); do { Options options = CurrentOptions(); + options.compression = kNoCompression; + BlockBasedTableOptions table_options; + table_options.block_size = 4 * 1024; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); std::string file_path = sst_files_dir_ + std::to_string(file_id++); SstFileWriter sst_file_writer(env_options, options); Status s = sst_file_writer.Open(file_path); ASSERT_OK(s); + // This should write more than 2 data blocks. for (int i = 0; i != 100; ++i) { std::string key = Key(i); - std::string value = Key(i) + std::to_string(0); + std::string value = rnd.RandomString(200); ASSERT_OK(sst_file_writer.Put(key, value)); } ASSERT_OK(sst_file_writer.Finish()); @@ -1609,11 +1692,11 @@ TEST_P(ExternalSSTFileBasicTest, IngestFileWithFirstByteTampered) { ASSERT_GT(file_size, 8); std::unique_ptr rwfile; ASSERT_OK(env_->NewRandomRWFile(file_path, &rwfile, EnvOptions())); - // Manually corrupt the file - // We deterministically corrupt the first byte because we currently - // cannot choose a random offset. The reason for this limitation is that - // we do not checksum property block at present. - const uint64_t offset = 0; + // Corrupt the second data block. + // We need to corrupt a non-first and non-last data block + // since we access them to get smallest and largest internal + // key in the file in GetIngestedFileInfo(). + const uint64_t offset = 5000; char scratch[8] = {0}; Slice buf; ASSERT_OK(rwfile->Read(offset, sizeof(scratch), &buf, scratch)); @@ -1747,11 +1830,11 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileAfterDBPut) { Options options = CurrentOptions(); ASSERT_OK(Put("k", "a")); - Flush(); + ASSERT_OK(Flush()); ASSERT_OK(Put("k", "a")); - Flush(); + ASSERT_OK(Flush()); ASSERT_OK(Put("k", "a")); - Flush(); + ASSERT_OK(Flush()); SstFileWriter sst_file_writer(EnvOptions(), options); // Current file size should be 0 after sst_file_writer init and before open a diff --git a/db/external_sst_file_ingestion_job.cc b/db/external_sst_file_ingestion_job.cc index c665ed83d..778e054c7 100644 --- a/db/external_sst_file_ingestion_job.cc +++ b/db/external_sst_file_ingestion_job.cc @@ -217,6 +217,8 @@ Status ExternalSstFileIngestionJob::Prepare( std::string requested_checksum_func_name; // TODO: rate limit file reads for checksum calculation during file // ingestion. + // TODO: plumb Env::IOActivity + ReadOptions ro; IOStatus io_s = GenerateOneFileChecksum( fs_.get(), files_to_ingest_[i].internal_file_path, db_options_.file_checksum_gen_factory.get(), @@ -224,8 +226,8 @@ Status ExternalSstFileIngestionJob::Prepare( &generated_checksum_func_name, ingestion_options_.verify_checksums_readahead_size, db_options_.allow_mmap_reads, io_tracer_, - db_options_.rate_limiter.get(), - Env::IO_TOTAL /* rate_limiter_priority */); + db_options_.rate_limiter.get(), ro, db_options_.stats, + db_options_.clock); if (!io_s.ok()) { status = io_s; ROCKS_LOG_WARN(db_options_.info_log, @@ -348,7 +350,7 @@ Status ExternalSstFileIngestionJob::NeedsFlush(bool* flush_needed, std::string end_str; AppendUserKeyWithMaxTimestamp( &begin_str, file_to_ingest.smallest_internal_key.user_key(), ts_sz); - AppendKeyWithMinTimestamp( + AppendUserKeyWithMinTimestamp( &end_str, file_to_ingest.largest_internal_key.user_key(), ts_sz); keys.emplace_back(std::move(begin_str)); keys.emplace_back(std::move(end_str)); @@ -767,8 +769,6 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo( std::unique_ptr iter(table_reader->NewIterator( ro, sv->mutable_cf_options.prefix_extractor.get(), /*arena=*/nullptr, /*skip_filters=*/false, TableReaderCaller::kExternalSSTIngestion)); - std::unique_ptr range_del_iter( - table_reader->NewRangeTombstoneIterator(ro)); // Get first (smallest) and last (largest) key from file. file_to_ingest->smallest_internal_key = @@ -790,8 +790,33 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo( } file_to_ingest->smallest_internal_key.SetFrom(key); - iter->SeekToLast(); - pik_status = ParseInternalKey(iter->key(), &key, allow_data_in_errors); + Slice largest; + if (strcmp(cfd_->ioptions()->table_factory->Name(), "PlainTable") == 0) { + // PlainTable iterator does not support SeekToLast(). + largest = iter->key(); + for (; iter->Valid(); iter->Next()) { + if (cfd_->internal_comparator().Compare(iter->key(), largest) > 0) { + largest = iter->key(); + } + } + if (!iter->status().ok()) { + return iter->status(); + } + } else { + iter->SeekToLast(); + if (!iter->Valid()) { + if (iter->status().ok()) { + // The file contains at least 1 key since iter is valid after + // SeekToFirst(). + return Status::Corruption("Can not find largest key in sst file"); + } else { + return iter->status(); + } + } + largest = iter->key(); + } + + pik_status = ParseInternalKey(largest, &key, allow_data_in_errors); if (!pik_status.ok()) { return Status::Corruption("Corrupted key in external file. ", pik_status.getState()); @@ -802,8 +827,12 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo( file_to_ingest->largest_internal_key.SetFrom(key); bounds_set = true; + } else if (!iter->status().ok()) { + return iter->status(); } + std::unique_ptr range_del_iter( + table_reader->NewRangeTombstoneIterator(ro)); // We may need to adjust these key bounds, depending on whether any range // deletion tombstones extend past them. const Comparator* ucmp = cfd_->internal_comparator().user_comparator(); @@ -908,26 +937,6 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile( overlap_with_db = true; break; } - - if (compaction_style == kCompactionStyleUniversal && lvl != 0) { - const std::vector& level_files = - vstorage->LevelFiles(lvl); - const SequenceNumber level_largest_seqno = - (*std::max_element(level_files.begin(), level_files.end(), - [](FileMetaData* f1, FileMetaData* f2) { - return f1->fd.largest_seqno < - f2->fd.largest_seqno; - })) - ->fd.largest_seqno; - // should only assign seqno to current level's largest seqno when - // the file fits - if (level_largest_seqno != 0 && - IngestedFileFitInLevel(file_to_ingest, lvl)) { - *assigned_seqno = level_largest_seqno; - } else { - continue; - } - } } else if (compaction_style == kCompactionStyleUniversal) { continue; } @@ -1058,13 +1067,15 @@ IOStatus ExternalSstFileIngestionJob::GenerateChecksumForIngestedFile( std::string file_checksum_func_name; std::string requested_checksum_func_name; // TODO: rate limit file reads for checksum calculation during file ingestion. + // TODO: plumb Env::IOActivity + ReadOptions ro; IOStatus io_s = GenerateOneFileChecksum( fs_.get(), file_to_ingest->internal_file_path, db_options_.file_checksum_gen_factory.get(), requested_checksum_func_name, &file_checksum, &file_checksum_func_name, ingestion_options_.verify_checksums_readahead_size, db_options_.allow_mmap_reads, io_tracer_, db_options_.rate_limiter.get(), - Env::IO_TOTAL /* rate_limiter_priority */); + ro, db_options_.stats, db_options_.clock); if (!io_s.ok()) { return io_s; } diff --git a/db/external_sst_file_test.cc b/db/external_sst_file_test.cc index 4249b2fc6..ef4ab7fa5 100644 --- a/db/external_sst_file_test.cc +++ b/db/external_sst_file_test.cc @@ -538,6 +538,113 @@ TEST_F(ExternalSSTFileTest, Basic) { kRangeDelSkipConfigs)); } +TEST_F(ExternalSSTFileTest, BasicWideColumn) { + do { + Options options = CurrentOptions(); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // Current file size should be 0 after sst_file_writer init and before open + // a file. + ASSERT_EQ(sst_file_writer.FileSize(), 0); + + std::string file = sst_files_dir_ + "wide_column_file.sst"; + ASSERT_OK(sst_file_writer.Open(file)); + for (int k = 0; k < 10; k++) { + std::string val1 = Key(k) + "_attr_1_val"; + std::string val2 = Key(k) + "_attr_2_val"; + WideColumns columns{{"attr_1", val1}, {"attr_2", val2}}; + ASSERT_OK(sst_file_writer.PutEntity(Key(k), columns)); + } + ExternalSstFileInfo file_info; + ASSERT_OK(sst_file_writer.Finish(&file_info)); + + // Current file size should be non-zero after success write. + ASSERT_GT(sst_file_writer.FileSize(), 0); + + ASSERT_EQ(file_info.file_path, file); + ASSERT_EQ(file_info.num_entries, 10); + ASSERT_EQ(file_info.smallest_key, Key(0)); + ASSERT_EQ(file_info.largest_key, Key(9)); + ASSERT_EQ(file_info.num_range_del_entries, 0); + ASSERT_EQ(file_info.smallest_range_del_key, ""); + ASSERT_EQ(file_info.largest_range_del_key, ""); + + DestroyAndReopen(options); + // Add file using file path + ASSERT_OK(DeprecatedAddFile({file})); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 10; k++) { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + Key(k), &result)); + std::string val1 = Key(k) + "_attr_1_val"; + std::string val2 = Key(k) + "_attr_2_val"; + WideColumns expected_columns{{"attr_1", val1}, {"attr_2", val2}}; + ASSERT_EQ(result.columns(), expected_columns); + } + + } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction | + kRangeDelSkipConfigs)); +} + +TEST_F(ExternalSSTFileTest, BasicMixed) { + do { + Options options = CurrentOptions(); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // Current file size should be 0 after sst_file_writer init and before open + // a file. + ASSERT_EQ(sst_file_writer.FileSize(), 0); + + std::string file = sst_files_dir_ + "mixed_file.sst"; + ASSERT_OK(sst_file_writer.Open(file)); + for (int k = 0; k < 100; k++) { + if (k % 5 == 0) { + std::string val1 = Key(k) + "_attr_1_val"; + std::string val2 = Key(k) + "_attr_2_val"; + WideColumns columns{{"attr_1", val1}, {"attr_2", val2}}; + ASSERT_OK(sst_file_writer.PutEntity(Key(k), columns)); + } else { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + } + ExternalSstFileInfo file_info; + ASSERT_OK(sst_file_writer.Finish(&file_info)); + + // Current file size should be non-zero after success write. + ASSERT_GT(sst_file_writer.FileSize(), 0); + + ASSERT_EQ(file_info.file_path, file); + ASSERT_EQ(file_info.num_entries, 100); + ASSERT_EQ(file_info.smallest_key, Key(0)); + ASSERT_EQ(file_info.largest_key, Key(99)); + ASSERT_EQ(file_info.num_range_del_entries, 0); + ASSERT_EQ(file_info.smallest_range_del_key, ""); + ASSERT_EQ(file_info.largest_range_del_key, ""); + + DestroyAndReopen(options); + // Add file using file path + ASSERT_OK(DeprecatedAddFile({file})); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 10; k++) { + if (k % 5 == 0) { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + Key(k), &result)); + std::string val1 = Key(k) + "_attr_1_val"; + std::string val2 = Key(k) + "_attr_2_val"; + WideColumns expected_columns{{"attr_1", val1}, {"attr_2", val2}}; + ASSERT_EQ(result.columns(), expected_columns); + } else { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } + } + } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction | + kRangeDelSkipConfigs)); +} + class SstFileWriterCollector : public TablePropertiesCollector { public: explicit SstFileWriterCollector(const std::string prefix) : prefix_(prefix) { @@ -2223,7 +2330,7 @@ TEST_P(ExternalSSTFileTest, IngestBehind) { // Trigger compaction if size amplification exceeds 110%. options.compaction_options_universal.max_size_amplification_percent = 110; options.level0_file_num_compaction_trigger = 4; - TryReopen(options); + ASSERT_OK(TryReopen(options)); Random rnd(301); for (int i = 0; i < 4; ++i) { for (int j = 0; j < 10; j++) { @@ -2239,7 +2346,7 @@ TEST_P(ExternalSSTFileTest, IngestBehind) { // Turning off the option allows DB to compact ingested files. options.allow_ingest_behind = false; - TryReopen(options); + ASSERT_OK(TryReopen(options)); ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files); ASSERT_EQ(1, level_to_files[2].size()); @@ -2516,6 +2623,7 @@ TEST_P(ExternalSSTFileTest, "AfterRead"); ingest_thread.join(); for (auto* iter : iters) { + ASSERT_OK(iter->status()); delete iter; } iters.clear(); diff --git a/db/fault_injection_test.cc b/db/fault_injection_test.cc index ddd4b47cc..d888dfde1 100644 --- a/db/fault_injection_test.cc +++ b/db/fault_injection_test.cc @@ -443,7 +443,7 @@ TEST_P(FaultInjectionTest, UninstalledCompaction) { options_.level0_stop_writes_trigger = 1 << 10; options_.level0_slowdown_writes_trigger = 1 << 10; options_.max_background_compactions = 1; - OpenDB(); + ASSERT_OK(OpenDB()); if (!sequential_order_) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ diff --git a/db/flush_job.cc b/db/flush_job.cc index 3854e967a..a3e168823 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -79,6 +79,8 @@ const char* GetFlushReasonString(FlushReason flush_reason) { return "Error Recovery Retry Flush"; case FlushReason::kWalFull: return "WAL Full"; + case FlushReason::kCatchUpAfterErrorRecovery: + return "Catch Up After Error Recovery"; default: return "Invalid"; } @@ -98,7 +100,7 @@ FlushJob::FlushJob( Statistics* stats, EventLogger* event_logger, bool measure_io_stats, const bool sync_output_directory, const bool write_manifest, Env::Priority thread_pri, const std::shared_ptr& io_tracer, - const SeqnoToTimeMapping& seqno_time_mapping, const std::string& db_id, + const SeqnoToTimeMapping& seqno_to_time_mapping, const std::string& db_id, const std::string& db_session_id, std::string full_history_ts_low, BlobFileCompletionCallback* blob_callback) : dbname_(dbname), @@ -134,7 +136,7 @@ FlushJob::FlushJob( clock_(db_options_.clock), full_history_ts_low_(std::move(full_history_ts_low)), blob_callback_(blob_callback), - db_impl_seqno_time_mapping_(seqno_time_mapping) { + db_impl_seqno_to_time_mapping_(seqno_to_time_mapping) { // Update the thread status to indicate flush. ReportStartedFlush(); TEST_SYNC_POINT("FlushJob::FlushJob()"); @@ -189,6 +191,10 @@ void FlushJob::PickMemTable() { return; } + // Track effective cutoff user-defined timestamp during flush if + // user-defined timestamps can be stripped. + GetEffectiveCutoffUDTForPickedMemTables(); + ReportFlushInputSize(mems_); // entries mems are (implicitly) sorted in ascending order by their created @@ -211,7 +217,8 @@ void FlushJob::PickMemTable() { } Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta, - bool* switched_to_mempurge) { + bool* switched_to_mempurge, bool* skipped_since_bg_error, + ErrorHandler* error_handler) { TEST_SYNC_POINT("FlushJob::Start"); db_mutex_->AssertHeld(); assert(pick_memtable_called); @@ -294,18 +301,37 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta, s = Status::ShutdownInProgress("Database shutdown"); } + if (s.ok()) { + s = MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT(); + } + if (!s.ok()) { - cfd_->imm()->RollbackMemtableFlush(mems_, meta_.fd.GetNumber()); + cfd_->imm()->RollbackMemtableFlush( + mems_, /*rollback_succeeding_memtables=*/!db_options_.atomic_flush); } else if (write_manifest_) { - TEST_SYNC_POINT("FlushJob::InstallResults"); - // Replace immutable memtable with the generated Table - s = cfd_->imm()->TryInstallMemtableFlushResults( - cfd_, mutable_cf_options_, mems_, prep_tracker, versions_, db_mutex_, - meta_.fd.GetNumber(), &job_context_->memtables_to_free, db_directory_, - log_buffer_, &committed_flush_jobs_info_, - !(mempurge_s.ok()) /* write_edit : true if no mempurge happened (or if aborted), + assert(!db_options_.atomic_flush); + if (!db_options_.atomic_flush && + flush_reason_ != FlushReason::kErrorRecovery && + flush_reason_ != FlushReason::kErrorRecoveryRetryFlush && + error_handler && !error_handler->GetBGError().ok() && + error_handler->IsBGWorkStopped()) { + cfd_->imm()->RollbackMemtableFlush( + mems_, /*rollback_succeeding_memtables=*/!db_options_.atomic_flush); + s = error_handler->GetBGError(); + if (skipped_since_bg_error) { + *skipped_since_bg_error = true; + } + } else { + TEST_SYNC_POINT("FlushJob::InstallResults"); + // Replace immutable memtable with the generated Table + s = cfd_->imm()->TryInstallMemtableFlushResults( + cfd_, mutable_cf_options_, mems_, prep_tracker, versions_, db_mutex_, + meta_.fd.GetNumber(), &job_context_->memtables_to_free, db_directory_, + log_buffer_, &committed_flush_jobs_info_, + !(mempurge_s.ok()) /* write_edit : true if no mempurge happened (or if aborted), but 'false' if mempurge successful: no new min log number or new level 0 file path to write to manifest. */); + } } if (s.ok() && file_meta != nullptr) { @@ -482,6 +508,7 @@ Status FlushJob::MemPurge() { nullptr, ioptions->allow_data_in_errors, ioptions->enforce_single_del_contracts, /*manual_compaction_canceled=*/kManualCompactionCanceledFalse, + false /* must_count_input_entries */, /*compaction=*/nullptr, compaction_filter.get(), /*shutting_down=*/nullptr, ioptions->info_log, full_history_ts_low); @@ -824,10 +851,11 @@ Status FlushJob::WriteLevel0Table() { Status s; SequenceNumber smallest_seqno = mems_.front()->GetEarliestSequenceNumber(); - if (!db_impl_seqno_time_mapping_.Empty()) { - // make a local copy, as the seqno_time_mapping from db_impl is not thread - // safe, which will be used while not holding the db_mutex. - seqno_to_time_mapping_ = db_impl_seqno_time_mapping_.Copy(smallest_seqno); + if (!db_impl_seqno_to_time_mapping_.Empty()) { + // make a local copy, as the seqno_to_time_mapping from db_impl is not + // thread safe, which will be used while not holding the db_mutex. + seqno_to_time_mapping_ = + db_impl_seqno_to_time_mapping_.Copy(smallest_seqno); } std::vector blob_file_additions; @@ -852,6 +880,7 @@ Status FlushJob::WriteLevel0Table() { uint64_t total_num_entries = 0, total_num_deletes = 0; uint64_t total_data_size = 0; size_t total_memory_usage = 0; + uint64_t total_num_range_deletes = 0; // Used for testing: uint64_t mems_size = mems_.size(); (void)mems_size; // avoids unused variable error when @@ -874,15 +903,20 @@ Status FlushJob::WriteLevel0Table() { total_num_deletes += m->num_deletes(); total_data_size += m->get_data_size(); total_memory_usage += m->ApproximateMemoryUsage(); + total_num_range_deletes += m->num_range_deletes(); } + // TODO(cbi): when memtable is flushed due to number of range deletions + // hitting limit memtable_max_range_deletions, flush_reason_ is still + // "Write Buffer Full", should make update flush_reason_ accordingly. event_logger_->Log() << "job" << job_context_->job_id << "event" << "flush_started" << "num_memtables" << mems_.size() << "num_entries" << total_num_entries << "num_deletes" << total_num_deletes << "total_data_size" << total_data_size << "memory_usage" - << total_memory_usage << "flush_reason" + << total_memory_usage << "num_range_deletes" + << total_num_range_deletes << "flush_reason" << GetFlushReasonString(flush_reason_); { @@ -950,6 +984,7 @@ Status FlushJob::WriteLevel0Table() { &table_properties_, write_hint, full_history_ts_low, blob_callback_, base_, &num_input_entries, &memtable_payload_bytes, &memtable_garbage_bytes); + TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:s", &s); // TODO: Cleanup io_status in BuildTable and table builders assert(!s.ok() || io_s.ok()); io_s.PermitUncheckedError(); @@ -1097,4 +1132,54 @@ std::unique_ptr FlushJob::GetFlushJobInfo() const { return info; } +void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() { + db_mutex_->AssertHeld(); + assert(pick_memtable_called); + const auto* ucmp = cfd_->internal_comparator().user_comparator(); + assert(ucmp); + const size_t ts_sz = ucmp->timestamp_size(); + if (db_options_.atomic_flush || ts_sz == 0 || + cfd_->ioptions()->persist_user_defined_timestamps) { + return; + } + // Find the newest user-defined timestamps from all the flushed memtables. + for (MemTable* m : mems_) { + Slice table_newest_udt = m->GetNewestUDT(); + if (cutoff_udt_.empty() || + ucmp->CompareTimestamp(table_newest_udt, cutoff_udt_) > 0) { + if (!cutoff_udt_.empty()) { + assert(table_newest_udt.size() == cutoff_udt_.size()); + } + cutoff_udt_.assign(table_newest_udt.data(), table_newest_udt.size()); + } + } +} + +Status FlushJob::MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT() { + db_mutex_->AssertHeld(); + const auto* ucmp = cfd_->user_comparator(); + assert(ucmp); + const std::string& full_history_ts_low = cfd_->GetFullHistoryTsLow(); + // Update full_history_ts_low to right above cutoff udt only if that would + // increase it. + if (cutoff_udt_.empty() || + (!full_history_ts_low.empty() && + ucmp->CompareTimestamp(cutoff_udt_, full_history_ts_low) < 0)) { + return Status::OK(); + } + std::string new_full_history_ts_low; + Slice cutoff_udt_slice = cutoff_udt_; + // TODO(yuzhangyu): Add a member to AdvancedColumnFamilyOptions for an + // operation to get the next immediately larger user-defined timestamp to + // expand this feature to other user-defined timestamp formats. + GetFullHistoryTsLowFromU64CutoffTs(&cutoff_udt_slice, + &new_full_history_ts_low); + VersionEdit edit; + edit.SetColumnFamily(cfd_->GetID()); + edit.SetFullHistoryTsLow(new_full_history_ts_low); + return versions_->LogAndApply(cfd_, *cfd_->GetLatestMutableCFOptions(), + ReadOptions(), &edit, db_mutex_, + output_file_directory_); +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/flush_job.h b/db/flush_job.h index d3902f0bd..aef33ef42 100644 --- a/db/flush_job.h +++ b/db/flush_job.h @@ -83,9 +83,14 @@ class FlushJob { // Require db_mutex held. // Once PickMemTable() is called, either Run() or Cancel() has to be called. void PickMemTable(); + // @param skip_since_bg_error If not nullptr and if atomic_flush=false, + // then it is set to true if flush installation is skipped and memtable + // is rolled back due to existing background error. Status Run(LogsWithPrepTracker* prep_tracker = nullptr, FileMetaData* file_meta = nullptr, - bool* switched_to_mempurge = nullptr); + bool* switched_to_mempurge = nullptr, + bool* skipped_since_bg_error = nullptr, + ErrorHandler* error_handler = nullptr); void Cancel(); const autovector& GetMemTables() const { return mems_; } @@ -127,6 +132,20 @@ class FlushJob { Env::IOPriority GetRateLimiterPriorityForWrite(); std::unique_ptr GetFlushJobInfo() const; + // Require db_mutex held. + // Called only when UDT feature is enabled and + // `persist_user_defined_timestamps` flag is false. Because we will refrain + // from flushing as long as there are still UDTs in a memtable that hasn't + // expired w.r.t `full_history_ts_low`. However, flush is continued if there + // is risk of entering write stall mode. In that case, we need + // to track the effective cutoff timestamp below which all the udts are + // removed because of flush, and use it to increase `full_history_ts_low` if + // the effective cutoff timestamp is newer. See + // `MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT` for details. + void GetEffectiveCutoffUDTForPickedMemTables(); + + Status MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT(); + const std::string& dbname_; const std::string db_id_; const std::string db_session_id_; @@ -191,10 +210,14 @@ class FlushJob { const std::string full_history_ts_low_; BlobFileCompletionCallback* blob_callback_; - // reference to the seqno_time_mapping_ in db_impl.h, not safe to read without - // db mutex - const SeqnoToTimeMapping& db_impl_seqno_time_mapping_; + // reference to the seqno_to_time_mapping_ in db_impl.h, not safe to read + // without db mutex + const SeqnoToTimeMapping& db_impl_seqno_to_time_mapping_; SeqnoToTimeMapping seqno_to_time_mapping_; + + // Keeps track of the newest user-defined timestamp for this flush job if + // `persist_user_defined_timestamps` flag is false. + std::string cutoff_udt_; }; } // namespace ROCKSDB_NAMESPACE diff --git a/db/flush_job_test.cc b/db/flush_job_test.cc index f2915ed39..21d1571a0 100644 --- a/db/flush_job_test.cc +++ b/db/flush_job_test.cc @@ -70,6 +70,7 @@ class FlushJobTestBase : public testing::Test { new_cf.AddColumnFamily(column_family_names_[i]); new_cf.SetColumnFamily(cf_id++); new_cf.SetComparatorName(ucmp_->Name()); + new_cf.SetPersistUserDefinedTimestamps(persist_udt_); new_cf.SetLogNumber(0); new_cf.SetNextFile(2); new_cf.SetLastSequence(last_seq++); @@ -126,11 +127,12 @@ class FlushJobTestBase : public testing::Test { column_families.emplace_back(cf_name, cf_options_); } - versions_.reset( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + versions_.reset(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); EXPECT_OK(versions_->Recover(column_families, false)); } @@ -456,7 +458,8 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) { // Verify that imm is empty ASSERT_EQ(std::numeric_limits::max(), all_cfds[k]->imm()->GetEarliestMemTableID()); - ASSERT_EQ(0, all_cfds[k]->imm()->GetLatestMemTableID()); + ASSERT_EQ(0, all_cfds[k]->imm()->GetLatestMemTableID( + false /* for_atomic_flush */)); ++k; } @@ -654,6 +657,10 @@ class FlushJobTimestampTest installed_file_meta->smallest.Encode()); ASSERT_EQ(expected_largest.Encode(), installed_file_meta->largest.Encode()); } + void CheckFullHistoryTsLow(ColumnFamilyData* cfd, + const std::string& expected_full_history_ts_low) { + ASSERT_EQ(expected_full_history_ts_low, cfd->GetFullHistoryTsLow()); + } }; TEST_P(FlushJobTimestampTest, AllKeysExpired) { @@ -684,6 +691,7 @@ TEST_P(FlushJobTimestampTest, AllKeysExpired) { EventLogger event_logger(db_options_.info_log.get()); std::string full_history_ts_low; PutFixed64(&full_history_ts_low, std::numeric_limits::max()); + cfd->SetFullHistoryTsLow(full_history_ts_low); FlushJob flush_job( dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(), std::numeric_limits::max() /* memtable_id */, env_options_, @@ -714,6 +722,7 @@ TEST_P(FlushJobTimestampTest, AllKeysExpired) { } InternalKey ikey(key, curr_seq_ - 1, ValueType::kTypeDeletionWithTimestamp); CheckFileMetaData(cfd, ikey, ikey, &fmeta); + CheckFullHistoryTsLow(cfd, full_history_ts_low); } job_context.Clean(); @@ -744,6 +753,7 @@ TEST_P(FlushJobTimestampTest, NoKeyExpired) { EventLogger event_logger(db_options_.info_log.get()); std::string full_history_ts_low; PutFixed64(&full_history_ts_low, 0); + cfd->SetFullHistoryTsLow(full_history_ts_low); FlushJob flush_job( dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(), std::numeric_limits::max() /* memtable_id */, env_options_, @@ -765,6 +775,7 @@ TEST_P(FlushJobTimestampTest, NoKeyExpired) { std::string ukey = test::EncodeInt(0); std::string smallest_key; std::string largest_key; + std::string expected_full_history_ts_low; if (!persist_udt_) { // When `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` flag // is set to false. The user-defined timestamp is stripped from user key @@ -772,14 +783,21 @@ TEST_P(FlushJobTimestampTest, NoKeyExpired) { // timestamp, which is hardcoded to be all zeros for now. smallest_key = ukey + test::EncodeInt(0); largest_key = ukey + test::EncodeInt(0); + // When not all keys have expired and `persist_user_defined_timestamps` is + // false. UDTs will be removed during flush, `full_history_ts_low` should + // be automatically increased to above the effective cutoff UDT in the + // flush. + PutFixed64(&expected_full_history_ts_low, curr_ts_.fetch_add(1)); } else { smallest_key = ukey + test::EncodeInt(curr_ts_.load(std::memory_order_relaxed) - 1); largest_key = ukey + test::EncodeInt(kStartTs); + expected_full_history_ts_low = full_history_ts_low; } InternalKey smallest(smallest_key, curr_seq_ - 1, ValueType::kTypeValue); InternalKey largest(largest_key, kStartSeq, ValueType::kTypeValue); CheckFileMetaData(cfd, smallest, largest, &fmeta); + CheckFullHistoryTsLow(cfd, expected_full_history_ts_low); } job_context.Clean(); ASSERT_TRUE(to_delete.empty()); diff --git a/db/import_column_family_job.cc b/db/import_column_family_job.cc index 11c5fd41b..4ef12d8fa 100644 --- a/db/import_column_family_job.cc +++ b/db/import_column_family_job.cc @@ -66,11 +66,11 @@ Status ImportColumnFamilyJob::Prepare(uint64_t next_file_number, largest = file_to_import.largest_internal_key; } else { if (cfd_->internal_comparator().Compare( - smallest, file_to_import.smallest_internal_key) < 0) { + smallest, file_to_import.smallest_internal_key) > 0) { smallest = file_to_import.smallest_internal_key; } if (cfd_->internal_comparator().Compare( - largest, file_to_import.largest_internal_key) > 0) { + largest, file_to_import.largest_internal_key) < 0) { largest = file_to_import.largest_internal_key; } } @@ -175,20 +175,29 @@ Status ImportColumnFamilyJob::Run() { static_cast(temp_current_time); } - // Recover files' epoch number using dummy VersionStorageInfo - VersionBuilder dummy_version_builder( - cfd_->current()->version_set()->file_options(), cfd_->ioptions(), - cfd_->table_cache(), cfd_->current()->storage_info(), - cfd_->current()->version_set(), - cfd_->GetFileMetadataCacheReservationManager()); - VersionStorageInfo dummy_vstorage( - &cfd_->internal_comparator(), cfd_->user_comparator(), - cfd_->NumberLevels(), cfd_->ioptions()->compaction_style, - nullptr /* src_vstorage */, cfd_->ioptions()->force_consistency_checks, - EpochNumberRequirement::kMightMissing); Status s; - + // When importing multiple CFs, we should not reuse epoch number from ingested + // files. Since these epoch numbers were assigned by different CFs, there may + // be different files from different CFs with the same epoch number. With a + // subsequent intra-L0 compaction we may end up with files with overlapping + // key range but the same epoch number. Here we will create a dummy + // VersionStorageInfo per CF being imported. Each CF's files will be assigned + // increasing epoch numbers to avoid duplicated epoch number. This is done by + // only resetting epoch number of the new CF in the first call to + // RecoverEpochNumbers() below. for (size_t i = 0; s.ok() && i < files_to_import_.size(); ++i) { + VersionBuilder dummy_version_builder( + cfd_->current()->version_set()->file_options(), cfd_->ioptions(), + cfd_->table_cache(), cfd_->current()->storage_info(), + cfd_->current()->version_set(), + cfd_->GetFileMetadataCacheReservationManager()); + VersionStorageInfo dummy_vstorage( + &cfd_->internal_comparator(), cfd_->user_comparator(), + cfd_->NumberLevels(), cfd_->ioptions()->compaction_style, + nullptr /* src_vstorage */, cfd_->ioptions()->force_consistency_checks, + EpochNumberRequirement::kMightMissing, cfd_->ioptions()->clock, + cfd_->GetLatestMutableCFOptions()->bottommost_file_compaction_delay, + cfd_->current()->version_set()->offpeak_time_option()); for (size_t j = 0; s.ok() && j < files_to_import_[i].size(); ++j) { const auto& f = files_to_import_[i][j]; const auto& file_metadata = *metadatas_[i][j]; @@ -216,42 +225,39 @@ Status ImportColumnFamilyJob::Run() { f.table_properties.user_defined_timestamps_persisted)); s = dummy_version_builder.Apply(&dummy_version_edit); } - } - - if (s.ok()) { - s = dummy_version_builder.SaveTo(&dummy_vstorage); - } - if (s.ok()) { - dummy_vstorage.RecoverEpochNumbers(cfd_); - } - - // Record changes from this CF import in VersionEdit, including files with - // recovered epoch numbers - if (s.ok()) { - edit_.SetColumnFamily(cfd_->GetID()); - + if (s.ok()) { + s = dummy_version_builder.SaveTo(&dummy_vstorage); + } + if (s.ok()) { + // force resetting epoch number for each file + dummy_vstorage.RecoverEpochNumbers(cfd_, /*restart_epoch=*/i == 0, + /*force=*/true); + edit_.SetColumnFamily(cfd_->GetID()); + + for (int level = 0; level < dummy_vstorage.num_levels(); level++) { + for (FileMetaData* file_meta : dummy_vstorage.LevelFiles(level)) { + edit_.AddFile(level, *file_meta); + // If incoming sequence number is higher, update local sequence + // number. + if (file_meta->fd.largest_seqno > versions_->LastSequence()) { + versions_->SetLastAllocatedSequence(file_meta->fd.largest_seqno); + versions_->SetLastPublishedSequence(file_meta->fd.largest_seqno); + versions_->SetLastSequence(file_meta->fd.largest_seqno); + } + } + } + } + // Release resources occupied by the dummy VersionStorageInfo for (int level = 0; level < dummy_vstorage.num_levels(); level++) { for (FileMetaData* file_meta : dummy_vstorage.LevelFiles(level)) { - edit_.AddFile(level, *file_meta); - // If incoming sequence number is higher, update local sequence number. - if (file_meta->fd.largest_seqno > versions_->LastSequence()) { - versions_->SetLastAllocatedSequence(file_meta->fd.largest_seqno); - versions_->SetLastPublishedSequence(file_meta->fd.largest_seqno); - versions_->SetLastSequence(file_meta->fd.largest_seqno); + file_meta->refs--; + if (file_meta->refs <= 0) { + delete file_meta; } } } } - // Release resources occupied by the dummy VersionStorageInfo - for (int level = 0; level < dummy_vstorage.num_levels(); level++) { - for (FileMetaData* file_meta : dummy_vstorage.LevelFiles(level)) { - file_meta->refs--; - if (file_meta->refs <= 0) { - delete file_meta; - } - } - } return s; } @@ -364,9 +370,35 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo( bool bound_set = false; if (iter->Valid()) { file_to_import->smallest_internal_key.DecodeFrom(iter->key()); - iter->SeekToLast(); - file_to_import->largest_internal_key.DecodeFrom(iter->key()); + Slice largest; + if (strcmp(cfd_->ioptions()->table_factory->Name(), "PlainTable") == 0) { + // PlainTable iterator does not support SeekToLast(). + largest = iter->key(); + for (; iter->Valid(); iter->Next()) { + if (cfd_->internal_comparator().Compare(iter->key(), largest) > 0) { + largest = iter->key(); + } + } + if (!iter->status().ok()) { + return iter->status(); + } + } else { + iter->SeekToLast(); + if (!iter->Valid()) { + if (iter->status().ok()) { + // The file contains at least 1 key since iter is valid after + // SeekToFirst(). + return Status::Corruption("Can not find largest key in sst file"); + } else { + return iter->status(); + } + } + largest = iter->key(); + } + file_to_import->largest_internal_key.DecodeFrom(largest); bound_set = true; + } else if (!iter->status().ok()) { + return iter->status(); } std::unique_ptr range_del_iter{ @@ -428,4 +460,4 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo( return status; } -} // namespace ROCKSDB_NAMESPACE \ No newline at end of file +} // namespace ROCKSDB_NAMESPACE diff --git a/db/import_column_family_test.cc b/db/import_column_family_test.cc index f6c1a0248..15cae03ed 100644 --- a/db/import_column_family_test.cc +++ b/db/import_column_family_test.cc @@ -326,7 +326,7 @@ TEST_F(ImportColumnFamilyTest, ImportSSTFileWriterFilesWithRangeTombstone) { const SstFileMetaData* file_meta = nullptr; for (const auto& level_meta : import_cf_meta.levels) { if (!level_meta.files.empty()) { - file_meta = &(level_meta.files[0]); + file_meta = level_meta.files.data(); break; } } @@ -389,7 +389,7 @@ TEST_F(ImportColumnFamilyTest, ImportExportedSSTFromAnotherCF) { *metadata_ptr_, &import_cfh2_)); ASSERT_NE(import_cfh2_, nullptr); delete metadata_ptr_; - metadata_ptr_ = NULL; + metadata_ptr_ = nullptr; std::string value1, value2; @@ -881,11 +881,177 @@ TEST_F(ImportColumnFamilyTest, ImportMultiColumnFamilyWithOverlap) { delete db_copy; ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); } + +TEST_F(ImportColumnFamilyTest, ImportMultiColumnFamilySeveralFilesWithOverlap) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + SstFileWriter sfw_cf1(EnvOptions(), options, handles_[1]); + const std::string file1_sst_name = "file1.sst"; + const std::string file1_sst = sst_files_dir_ + file1_sst_name; + ASSERT_OK(sfw_cf1.Open(file1_sst)); + ASSERT_OK(sfw_cf1.Put("K1", "V1")); + ASSERT_OK(sfw_cf1.Put("K2", "V2")); + ASSERT_OK(sfw_cf1.Finish()); + + SstFileWriter sfw_cf2(EnvOptions(), options, handles_[1]); + const std::string file2_sst_name = "file2.sst"; + const std::string file2_sst = sst_files_dir_ + file2_sst_name; + ASSERT_OK(sfw_cf2.Open(file2_sst)); + ASSERT_OK(sfw_cf2.Put("K2", "V2")); + ASSERT_OK(sfw_cf2.Put("K3", "V3")); + ASSERT_OK(sfw_cf2.Finish()); + + ColumnFamilyHandle* second_cfh = nullptr; + ASSERT_OK(db_->CreateColumnFamily(options, "toto", &second_cfh)); + + SstFileWriter sfw_cf3(EnvOptions(), options, second_cfh); + const std::string file3_sst_name = "file3.sst"; + const std::string file3_sst = sst_files_dir_ + file3_sst_name; + ASSERT_OK(sfw_cf3.Open(file3_sst)); + ASSERT_OK(sfw_cf3.Put("K3", "V3")); + ASSERT_OK(sfw_cf3.Put("K4", "V4")); + ASSERT_OK(sfw_cf3.Finish()); + + SstFileWriter sfw_cf4(EnvOptions(), options, second_cfh); + const std::string file4_sst_name = "file4.sst"; + const std::string file4_sst = sst_files_dir_ + file4_sst_name; + ASSERT_OK(sfw_cf4.Open(file4_sst)); + ASSERT_OK(sfw_cf4.Put("K4", "V4")); + ASSERT_OK(sfw_cf4.Put("K5", "V5")); + ASSERT_OK(sfw_cf4.Finish()); + + ExportImportFilesMetaData metadata1, metadata2; + metadata1.files.push_back( + LiveFileMetaDataInit(file1_sst_name, sst_files_dir_, 1, 1, 2)); + metadata1.files.push_back( + LiveFileMetaDataInit(file2_sst_name, sst_files_dir_, 1, 3, 4)); + metadata1.db_comparator_name = options.comparator->Name(); + metadata2.files.push_back( + LiveFileMetaDataInit(file3_sst_name, sst_files_dir_, 1, 1, 2)); + metadata2.files.push_back( + LiveFileMetaDataInit(file4_sst_name, sst_files_dir_, 1, 3, 4)); + metadata2.db_comparator_name = options.comparator->Name(); + + std::vector metadatas{&metadata1, + &metadata2}; + + ASSERT_EQ(db_->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "yoyo", + ImportColumnFamilyOptions(), + metadatas, &import_cfh_), + Status::InvalidArgument("CFs have overlapping ranges")); + ASSERT_EQ(import_cfh_, nullptr); + + ASSERT_OK(db_->DropColumnFamily(second_cfh)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(second_cfh)); +} + +TEST_F(ImportColumnFamilyTest, AssignEpochNumberToMultipleCF) { + // Test ingesting CFs where L0 files could have the same epoch number. + Options options = CurrentOptions(); + options.level_compaction_dynamic_level_bytes = true; + options.max_background_jobs = 8; + env_->SetBackgroundThreads(2, Env::LOW); + env_->SetBackgroundThreads(0, Env::BOTTOM); + CreateAndReopenWithCF({"CF1", "CF2"}, options); + + // CF1: + // L6: [0, 99], [100, 199] + // CF2: + // L6: [1000, 1099], [1100, 1199] + for (int i = 100; i < 200; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + ASSERT_OK(Put(2, Key(1000 + i), Key(1000 + i) + "_val")); + } + ASSERT_OK(Flush(1)); + ASSERT_OK(Flush(2)); + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + ASSERT_OK(Put(2, Key(1000 + i), Key(1000 + i) + "_val")); + } + ASSERT_OK(Flush(1)); + ASSERT_OK(Flush(2)); + MoveFilesToLevel(6, 1); + MoveFilesToLevel(6, 2); + + // CF1: + // level 0 epoch: 5 file num 30 smallest key000010 - key000019 + // level 0 epoch: 4 file num 27 smallest key000000 - key000009 + // level 0 epoch: 3 file num 23 smallest key000100 - key000199 + // level 6 epoch: 2 file num 20 smallest key000000 - key000099 + // level 6 epoch: 1 file num 17 smallest key000100 - key000199 + // CF2: + // level 0 epoch: 5 file num 31 smallest key001010 - key001019 + // level 0 epoch: 4 file num 28 smallest key001000 - key001009 + // level 0 epoch: 3 file num 25 smallest key001020 - key001029 + // level 6 epoch: 2 file num 21 smallest key001000 - key001099 + // level 6 epoch: 1 file num 18 smallest key001100 - key001199 + for (int i = 100; i < 200; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + } + ASSERT_OK(Flush(1)); + for (int i = 20; i < 30; ++i) { + ASSERT_OK(Put(2, Key(i + 1000), Key(i + 1000) + "_val")); + } + ASSERT_OK(Flush(2)); + + for (int i = 0; i < 20; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + ASSERT_OK(Put(2, Key(i + 1000), Key(i + 1000) + "_val")); + if (i % 10 == 9) { + ASSERT_OK(Flush(1)); + ASSERT_OK(Flush(2)); + } + } + ASSERT_OK(Flush(1)); + ASSERT_OK(Flush(2)); + + // Create a CF by importing these two CF1 and CF2. + // Then two compactions will be triggerred, one to compact from L0 + // to L6 (files #23 and #17), and another to do intra-L0 compaction + // for the rest of the L0 files. Before a bug fix, we used to + // directly use the epoch numbers from the ingested files in the new CF. + // This means different files from different CFs can have the same epoch + // number. If the intra-L0 compaction finishes first, it can cause a + // corruption where two L0 files can have the same epoch number but + // with overlapping key range. + Checkpoint* checkpoint1; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint1)); + ASSERT_OK(checkpoint1->ExportColumnFamily(handles_[1], export_files_dir_, + &metadata_ptr_)); + ASSERT_OK(checkpoint1->ExportColumnFamily(handles_[2], export_files_dir2_, + &metadata_ptr2_)); + ASSERT_NE(metadata_ptr_, nullptr); + ASSERT_NE(metadata_ptr2_, nullptr); + + std::atomic_int compaction_counter = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:BeforeRun", + [&compaction_counter](void*) { + compaction_counter++; + if (compaction_counter == 1) { + // Wait for the next compaction to finish + TEST_SYNC_POINT("WaitForSecondCompaction"); + } + }); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCompaction:AfterCompaction", + "WaitForSecondCompaction"}}); + SyncPoint::GetInstance()->EnableProcessing(); + ImportColumnFamilyOptions import_options; + import_options.move_files = false; + std::vector metadatas = {metadata_ptr_, + metadata_ptr2_}; + ASSERT_OK(db_->CreateColumnFamilyWithImport(options, "CF3", import_options, + metadatas, &import_cfh_)); + WaitForCompactOptions o; + ASSERT_OK(db_->WaitForCompact(o)); + delete checkpoint1; +} } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); -} - +} \ No newline at end of file diff --git a/db/job_context.h b/db/job_context.h index a550ba2d4..48728f48d 100644 --- a/db/job_context.h +++ b/db/job_context.h @@ -15,6 +15,7 @@ #include "db/column_family.h" #include "db/log_writer.h" #include "db/version_set.h" +#include "util/autovector.h" namespace ROCKSDB_NAMESPACE { @@ -170,6 +171,16 @@ struct JobContext { // will be reused later std::vector log_recycle_files; + // Files quarantined from deletion. This list contains file numbers for files + // that are in an ambiguous states. This includes newly generated SST files + // and blob files from flush and compaction job whose VersionEdits' persist + // state in Manifest are unclear. An old manifest file whose immediately + // following new manifest file's CURRENT file creation is in an unclear state. + // WAL logs don't have this premature deletion risk since + // min_log_number_to_keep is only updated after successful manifest commits. + // So this data structure doesn't track log files. + autovector files_to_quarantine; + // a list of manifest files that we need to delete std::vector manifest_delete_files; diff --git a/db/listener_test.cc b/db/listener_test.cc index 206dba973..d298a86e7 100644 --- a/db/listener_test.cc +++ b/db/listener_test.cc @@ -132,8 +132,8 @@ class TestCompactionListener : public EventListener { ASSERT_EQ(db->GetEnv()->GetThreadID(), ci.thread_id); ASSERT_GT(ci.thread_id, 0U); - for (auto fl : {ci.input_files, ci.output_files}) { - for (auto fn : fl) { + for (const auto& fl : {ci.input_files, ci.output_files}) { + for (const auto& fn : fl) { auto it = ci.table_properties.find(fn); ASSERT_NE(it, ci.table_properties.end()); auto tp = it->second; @@ -237,7 +237,7 @@ class TestFlushListener : public EventListener { std::vector thread_list; ASSERT_OK(env_->GetThreadList(&thread_list)); bool found_match = false; - for (auto thread_status : thread_list) { + for (const auto& thread_status : thread_list) { if (thread_status.operation_type == ThreadStatus::OP_FLUSH || thread_status.operation_type == ThreadStatus::OP_COMPACTION) { if (thread_id == thread_status.thread_id) { @@ -582,7 +582,7 @@ TEST_F(EventListenerTest, CompactionReasonLevel) { for (int k = 1; k <= 30; k++) { ASSERT_OK(Put(Key(k), Key(k))); if (k % 10 == 0) { - Flush(); + ASSERT_OK(Flush()); } } @@ -893,7 +893,7 @@ class MemTableSealedListener : public EventListener { SequenceNumber latest_seq_number_; public: - MemTableSealedListener() {} + MemTableSealedListener() = default; void OnMemTableSealed(const MemTableInfo& info) override { latest_seq_number_ = info.first_seqno; } diff --git a/db/log_reader.cc b/db/log_reader.cc index 5ec262dcd..ca2f9c55b 100644 --- a/db/log_reader.cc +++ b/db/log_reader.cc @@ -9,7 +9,7 @@ #include "db/log_reader.h" -#include +#include #include "file/sequence_file_reader.h" #include "port/lang.h" @@ -21,7 +21,7 @@ namespace ROCKSDB_NAMESPACE { namespace log { -Reader::Reporter::~Reporter() {} +Reader::Reporter::~Reporter() = default; Reader::Reader(std::shared_ptr info_log, std::unique_ptr&& _file, @@ -469,12 +469,14 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size, const unsigned int type = header[6]; const uint32_t length = a | (b << 8); int header_size = kHeaderSize; - if ((type >= kRecyclableFullType && type <= kRecyclableLastType) || - type == kRecyclableUserDefinedTimestampSizeType) { + const bool is_recyclable_type = + ((type >= kRecyclableFullType && type <= kRecyclableLastType) || + type == kRecyclableUserDefinedTimestampSizeType); + if (is_recyclable_type) { + header_size = kRecyclableHeaderSize; if (end_of_buffer_offset_ - buffer_.size() == 0) { recycled_ = true; } - header_size = kRecyclableHeaderSize; // We need enough for the larger header if (buffer_.size() < static_cast(kRecyclableHeaderSize)) { int r = kEof; @@ -483,11 +485,8 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size, } continue; } - const uint32_t log_num = DecodeFixed32(header + 7); - if (log_num != log_number_) { - return kOldRecord; - } } + if (header_size + length > buffer_.size()) { assert(buffer_.size() >= static_cast(header_size)); *drop_size = buffer_.size(); @@ -499,6 +498,14 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size, return kBadRecordLen; } + if (is_recyclable_type) { + const uint32_t log_num = DecodeFixed32(header + 7); + if (log_num != log_number_) { + buffer_.remove_prefix(header_size + length); + return kOldRecord; + } + } + if (type == kZeroType && length == 0) { // Skip zero length record without reporting any drops since // such records are produced by the mmap based writing code in diff --git a/db/log_test.cc b/db/log_test.cc index fa5e2aa0f..0bf3bf5ae 100644 --- a/db/log_test.cc +++ b/db/log_test.cc @@ -1167,7 +1167,7 @@ TEST_P(StreamingCompressionTest, Basic) { } allocator->Deallocate((void*)output_buffer); } while (remaining > 0); - std::string uncompressed_buffer = ""; + std::string uncompressed_buffer; int ret_val = 0; size_t output_pos; char* uncompressed_output_buffer = (char*)allocator->Allocate(kBlockSize); diff --git a/db/log_writer.cc b/db/log_writer.cc index 86e0286cc..5fc46b33f 100644 --- a/db/log_writer.cc +++ b/db/log_writer.cc @@ -9,7 +9,7 @@ #include "db/log_writer.h" -#include +#include #include "file/writable_file_writer.h" #include "rocksdb/env.h" diff --git a/db/malloc_stats.cc b/db/malloc_stats.cc index 641e01f9a..33f19725d 100644 --- a/db/malloc_stats.cc +++ b/db/malloc_stats.cc @@ -9,8 +9,7 @@ #include "db/malloc_stats.h" -#include - +#include #include #include "port/jemalloc_helper.h" diff --git a/db/manual_compaction_test.cc b/db/manual_compaction_test.cc index e9767ab99..e84031065 100644 --- a/db/manual_compaction_test.cc +++ b/db/manual_compaction_test.cc @@ -58,7 +58,7 @@ class ManualCompactionTest : public testing::Test { class DestroyAllCompactionFilter : public CompactionFilter { public: - DestroyAllCompactionFilter() {} + DestroyAllCompactionFilter() = default; bool Filter(int /*level*/, const Slice& /*key*/, const Slice& existing_value, std::string* /*new_value*/, @@ -124,6 +124,7 @@ TEST_F(ManualCompactionTest, CompactTouchesAllKeys) { ASSERT_EQ("key3", itr->key().ToString()); itr->Next(); ASSERT_TRUE(!itr->Valid()); + ASSERT_OK(itr->status()); delete itr; delete options.compaction_filter; @@ -179,6 +180,7 @@ TEST_F(ManualCompactionTest, Test) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { num_keys++; } + ASSERT_OK(iter->status()); delete iter; ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; diff --git a/db/memtable.cc b/db/memtable.cc index dfef13a15..0b8786bc2 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -95,6 +95,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp, data_size_(0), num_entries_(0), num_deletes_(0), + num_range_deletes_(0), write_buffer_size_(mutable_cf_options.write_buffer_size), flush_in_progress_(false), flush_completed_(false), @@ -114,7 +115,9 @@ MemTable::MemTable(const InternalKeyComparator& cmp, ioptions.memtable_insert_with_hint_prefix_extractor.get()), oldest_key_time_(std::numeric_limits::max()), atomic_flush_seqno_(kMaxSequenceNumber), - approximate_memory_usage_(0) { + approximate_memory_usage_(0), + memtable_max_range_deletions_( + mutable_cf_options.memtable_max_range_deletions) { UpdateFlushState(); // something went wrong if we need to flush before inserting anything assert(!ShouldScheduleFlush()); @@ -143,6 +146,10 @@ MemTable::MemTable(const InternalKeyComparator& cmp, new_cache.get()), std::memory_order_relaxed); } + const Comparator* ucmp = cmp.user_comparator(); + assert(ucmp); + ts_sz_ = ucmp->timestamp_size(); + persist_user_defined_timestamps_ = ioptions.persist_user_defined_timestamps; } MemTable::~MemTable() { @@ -170,6 +177,14 @@ size_t MemTable::ApproximateMemoryUsage() { } bool MemTable::ShouldFlushNow() { + // This is set if memtable_max_range_deletions is > 0, + // and that many range deletions are done + if (memtable_max_range_deletions_ > 0 && + num_range_deletes_.load(std::memory_order_relaxed) >= + static_cast(memtable_max_range_deletions_)) { + return true; + } + size_t write_buffer_size = write_buffer_size_.load(std::memory_order_relaxed); // In a lot of times, we cannot allocate arena blocks that exactly matches the // buffer size. Thus we have to decide if we should over-allocate or @@ -357,7 +372,8 @@ class MemTableIterator : public InternalIterator { !mem.GetImmutableMemTableOptions()->inplace_update_support), protection_bytes_per_key_(mem.moptions_.protection_bytes_per_key), status_(Status::OK()), - logger_(mem.moptions_.info_log) { + logger_(mem.moptions_.info_log), + ts_sz_(mem.ts_sz_) { if (use_range_del_table) { iter_ = mem.range_del_table_->GetIterator(arena); } else if (prefix_extractor_ != nullptr && !read_options.total_order_seek && @@ -400,8 +416,7 @@ class MemTableIterator : public InternalIterator { PERF_COUNTER_ADD(seek_on_memtable_count, 1); if (bloom_) { // iterator should only use prefix bloom filter - auto ts_sz = comparator_.comparator.user_comparator()->timestamp_size(); - Slice user_k_without_ts(ExtractUserKeyAndStripTimestamp(k, ts_sz)); + Slice user_k_without_ts(ExtractUserKeyAndStripTimestamp(k, ts_sz_)); if (prefix_extractor_->InDomain(user_k_without_ts)) { if (!bloom_->MayContain( prefix_extractor_->Transform(user_k_without_ts))) { @@ -421,8 +436,7 @@ class MemTableIterator : public InternalIterator { PERF_TIMER_GUARD(seek_on_memtable_time); PERF_COUNTER_ADD(seek_on_memtable_count, 1); if (bloom_) { - auto ts_sz = comparator_.comparator.user_comparator()->timestamp_size(); - Slice user_k_without_ts(ExtractUserKeyAndStripTimestamp(k, ts_sz)); + Slice user_k_without_ts(ExtractUserKeyAndStripTimestamp(k, ts_sz_)); if (prefix_extractor_->InDomain(user_k_without_ts)) { if (!bloom_->MayContain( prefix_extractor_->Transform(user_k_without_ts))) { @@ -512,6 +526,7 @@ class MemTableIterator : public InternalIterator { uint32_t protection_bytes_per_key_; Status status_; Logger* logger_; + size_t ts_sz_; void VerifyEntryChecksum() { if (protection_bytes_per_key_ > 0 && Valid()) { @@ -625,8 +640,7 @@ Status MemTable::VerifyEncodedEntry(Slice encoded, if (!GetVarint32(&encoded, &ikey_len)) { return Status::Corruption("Unable to parse internal key length"); } - size_t ts_sz = GetInternalKeyComparator().user_comparator()->timestamp_size(); - if (ikey_len < 8 + ts_sz) { + if (ikey_len < 8 + ts_sz_) { return Status::Corruption("Internal key length too short"); } if (ikey_len > encoded.size()) { @@ -725,8 +739,7 @@ Status MemTable::Add(SequenceNumber s, ValueType type, } } - size_t ts_sz = GetInternalKeyComparator().user_comparator()->timestamp_size(); - Slice key_without_ts = StripTimestampFromUserKey(key, ts_sz); + Slice key_without_ts = StripTimestampFromUserKey(key, ts_sz_); if (!allow_concurrent) { // Extract prefix for insert with hint. @@ -754,6 +767,9 @@ Status MemTable::Add(SequenceNumber s, ValueType type, type == kTypeDeletionWithTimestamp) { num_deletes_.store(num_deletes_.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + } else if (type == kTypeRangeDeletion) { + uint64_t val = num_range_deletes_.load(std::memory_order_relaxed) + 1; + num_range_deletes_.store(val, std::memory_order_relaxed); } if (bloom_filter_ && prefix_extractor_ && @@ -776,6 +792,9 @@ Status MemTable::Add(SequenceNumber s, ValueType type, assert(first_seqno_.load() >= earliest_seqno_.load()); } assert(post_process_info == nullptr); + // TODO(yuzhangyu): support updating newest UDT for when `allow_concurrent` + // is true. + MaybeUpdateNewestUDT(key_slice); UpdateFlushState(); } else { bool res = (hint == nullptr) @@ -817,6 +836,7 @@ Status MemTable::Add(SequenceNumber s, ValueType type, auto new_cache = std::make_shared(); size_t size = cached_range_tombstone_.Size(); if (allow_concurrent) { + post_process_info->num_range_deletes++; range_del_mutex_.lock(); } for (size_t i = 0; i < size; ++i) { @@ -835,6 +855,7 @@ Status MemTable::Add(SequenceNumber s, ValueType type, new_local_cache_ref, new_cache.get()), std::memory_order_relaxed); } + if (allow_concurrent) { range_del_mutex_.unlock(); } @@ -1033,25 +1054,15 @@ static bool SaveValue(void* arg, const char* entry) { assert(s->do_merge); if (s->value || s->columns) { - std::string result; // `op_failure_scope` (an output parameter) is not provided (set to // nullptr) since a failure must be propagated regardless of its // value. *(s->status) = MergeHelper::TimedFullMerge( - merge_operator, s->key->user_key(), &v, - merge_context->GetOperands(), &result, s->logger, s->statistics, - s->clock, /* result_operand */ nullptr, - /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - - if (s->status->ok()) { - if (s->value) { - *(s->value) = std::move(result); - } else { - assert(s->columns); - s->columns->SetPlainValue(std::move(result)); - } - } + merge_operator, s->key->user_key(), + MergeHelper::kPlainBaseValue, v, merge_context->GetOperands(), + s->logger, s->statistics, s->clock, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + s->value, s->columns); } } else if (s->value) { s->value->assign(v.data(), v.size()); @@ -1096,35 +1107,15 @@ static bool SaveValue(void* arg, const char* entry) { } else if (*(s->merge_in_progress)) { assert(s->do_merge); - if (s->value) { - Slice value_of_default; - *(s->status) = WideColumnSerialization::GetValueOfDefaultColumn( - v, value_of_default); - if (s->status->ok()) { - // `op_failure_scope` (an output parameter) is not provided (set - // to nullptr) since a failure must be propagated regardless of - // its value. - *(s->status) = MergeHelper::TimedFullMerge( - merge_operator, s->key->user_key(), &value_of_default, - merge_context->GetOperands(), s->value, s->logger, - s->statistics, s->clock, /* result_operand */ nullptr, - /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - } - } else if (s->columns) { - std::string result; - // `op_failure_scope` (an output parameter) is not provided (set to - // nullptr) since a failure must be propagated regardless of its - // value. - *(s->status) = MergeHelper::TimedFullMergeWithEntity( - merge_operator, s->key->user_key(), v, - merge_context->GetOperands(), &result, s->logger, s->statistics, + if (s->value || s->columns) { + // `op_failure_scope` (an output parameter) is not provided (set + // to nullptr) since a failure must be propagated regardless of + // its value. + *(s->status) = MergeHelper::TimedFullMerge( + merge_operator, s->key->user_key(), MergeHelper::kWideBaseValue, + v, merge_context->GetOperands(), s->logger, s->statistics, s->clock, /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - - if (s->status->ok()) { - *(s->status) = s->columns->SetWideColumnValue(std::move(result)); - } + /* op_failure_scope */ nullptr, s->value, s->columns); } } else if (s->value) { Slice value_of_default; @@ -1155,25 +1146,14 @@ static bool SaveValue(void* arg, const char* entry) { case kTypeRangeDeletion: { if (*(s->merge_in_progress)) { if (s->value || s->columns) { - std::string result; // `op_failure_scope` (an output parameter) is not provided (set to // nullptr) since a failure must be propagated regardless of its // value. *(s->status) = MergeHelper::TimedFullMerge( - merge_operator, s->key->user_key(), nullptr, - merge_context->GetOperands(), &result, s->logger, s->statistics, - s->clock, /* result_operand */ nullptr, - /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - - if (s->status->ok()) { - if (s->value) { - *(s->value) = std::move(result); - } else { - assert(s->columns); - s->columns->SetPlainValue(std::move(result)); - } - } + merge_operator, s->key->user_key(), MergeHelper::kNoBaseValue, + merge_context->GetOperands(), s->logger, s->statistics, + s->clock, /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr, s->value, s->columns); } else { // We have found a final value (a base deletion) and have newer // merge operands that we do not intend to merge. Nothing remains @@ -1206,25 +1186,14 @@ static bool SaveValue(void* arg, const char* entry) { if (s->do_merge && merge_operator->ShouldMerge( merge_context->GetOperandsDirectionBackward())) { if (s->value || s->columns) { - std::string result; // `op_failure_scope` (an output parameter) is not provided (set to // nullptr) since a failure must be propagated regardless of its // value. *(s->status) = MergeHelper::TimedFullMerge( - merge_operator, s->key->user_key(), nullptr, - merge_context->GetOperands(), &result, s->logger, s->statistics, - s->clock, /* result_operand */ nullptr, - /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - - if (s->status->ok()) { - if (s->value) { - *(s->value) = std::move(result); - } else { - assert(s->columns); - s->columns->SetPlainValue(std::move(result)); - } - } + merge_operator, s->key->user_key(), MergeHelper::kNoBaseValue, + merge_context->GetOperands(), s->logger, s->statistics, + s->clock, /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr, s->value, s->columns); } *(s->found_final_value) = true; @@ -1263,6 +1232,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, // Avoiding recording stats for speed. return false; } + PERF_TIMER_GUARD(get_from_memtable_time); std::unique_ptr range_del_iter( @@ -1286,8 +1256,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, bool found_final_value = false; bool merge_in_progress = s->IsMergeInProgress(); bool may_contain = true; - size_t ts_sz = GetInternalKeyComparator().user_comparator()->timestamp_size(); - Slice user_key_without_ts = StripTimestampFromUserKey(key.user_key(), ts_sz); + Slice user_key_without_ts = StripTimestampFromUserKey(key.user_key(), ts_sz_); bool bloom_checked = false; if (bloom_filter_) { // when both memtable_whole_key_filtering and prefix_extractor_ are set, @@ -1393,7 +1362,7 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range, range_indexes[num_keys++] = iter.index(); } } - bloom_filter_->MayContain(num_keys, &bloom_keys[0], &may_match[0]); + bloom_filter_->MayContain(num_keys, bloom_keys.data(), may_match.data()); for (int i = 0; i < num_keys; ++i) { if (!may_match[i]) { temp_range.SkipIndex(range_indexes[i]); @@ -1672,4 +1641,22 @@ uint64_t MemTable::GetMinLogContainingPrepSection() { return min_prep_log_referenced_.load(); } +void MemTable::MaybeUpdateNewestUDT(const Slice& user_key) { + if (ts_sz_ == 0 || persist_user_defined_timestamps_) { + return; + } + const Comparator* ucmp = GetInternalKeyComparator().user_comparator(); + Slice udt = ExtractTimestampFromUserKey(user_key, ts_sz_); + if (newest_udt_.empty() || ucmp->CompareTimestamp(udt, newest_udt_) > 0) { + newest_udt_ = udt; + } +} + +const Slice& MemTable::GetNewestUDT() const { + // This path should not be invoked for MemTables that does not enable the UDT + // in Memtable only feature. + assert(ts_sz_ > 0 && !persist_user_defined_timestamps_); + return newest_udt_; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/memtable.h b/db/memtable.h index a461d908b..c55b34761 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -68,6 +68,7 @@ struct MemTablePostProcessInfo { uint64_t data_size = 0; uint64_t num_entries = 0; uint64_t num_deletes = 0; + uint64_t num_range_deletes = 0; }; using MultiGetRange = MultiGetContext::Range; @@ -332,6 +333,10 @@ class MemTable { num_deletes_.fetch_add(update_counters.num_deletes, std::memory_order_relaxed); } + if (update_counters.num_range_deletes > 0) { + num_range_deletes_.fetch_add(update_counters.num_range_deletes, + std::memory_order_relaxed); + } UpdateFlushState(); } @@ -349,10 +354,21 @@ class MemTable { return num_deletes_.load(std::memory_order_relaxed); } + // Get total number of range deletions in the mem table. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + uint64_t num_range_deletes() const { + return num_range_deletes_.load(std::memory_order_relaxed); + } + uint64_t get_data_size() const { return data_size_.load(std::memory_order_relaxed); } + size_t write_buffer_size() const { + return write_buffer_size_.load(std::memory_order_relaxed); + } + // Dynamically change the memtable's capacity. If set below the current usage, // the next key added will trigger a flush. Can only increase size when // memtable prefix bloom is disabled, since we can't easily allocate more @@ -527,6 +543,14 @@ class MemTable { } } + // Get the newest user-defined timestamp contained in this MemTable. Check + // `newest_udt_` for what newer means. This method should only be invoked for + // an MemTable that has enabled user-defined timestamp feature and set + // `persist_user_defined_timestamps` to false. The tracked newest UDT will be + // used by flush job in the background to help check the MemTable's + // eligibility for Flush. + const Slice& GetNewestUDT() const; + // Returns Corruption status if verification fails. static Status VerifyEntryChecksum(const char* entry, uint32_t protection_bytes_per_key, @@ -553,6 +577,7 @@ class MemTable { std::atomic data_size_; std::atomic num_entries_; std::atomic num_deletes_; + std::atomic num_range_deletes_; // Dynamically changeable memtable option std::atomic write_buffer_size_; @@ -614,9 +639,26 @@ class MemTable { // Gets refreshed inside `ApproximateMemoryUsage()` or `ShouldFlushNow` std::atomic approximate_memory_usage_; + // max range deletions in a memtable, before automatic flushing, 0 for + // unlimited. + uint32_t memtable_max_range_deletions_ = 0; + // Flush job info of the current memtable. std::unique_ptr flush_job_info_; + // Size in bytes for the user-defined timestamps. + size_t ts_sz_; + + // Whether to persist user-defined timestamps + bool persist_user_defined_timestamps_; + + // Newest user-defined timestamp contained in this MemTable. For ts1, and ts2 + // if Comparator::CompareTimestamp(ts1, ts2) > 0, ts1 is considered newer than + // ts2. We track this field for a MemTable if its column family has UDT + // feature enabled and the `persist_user_defined_timestamp` flag is false. + // Otherwise, this field just contains an empty Slice. + Slice newest_udt_; + // Updates flush_state_ using ShouldFlushNow() void UpdateFlushState(); @@ -653,6 +695,8 @@ class MemTable { void UpdateEntryChecksum(const ProtectionInfoKVOS64* kv_prot_info, const Slice& key, const Slice& value, ValueType type, SequenceNumber s, char* checksum_ptr); + + void MaybeUpdateNewestUDT(const Slice& user_key); }; extern const char* EncodeKey(std::string* scratch, const Slice& target); diff --git a/db/memtable_list.cc b/db/memtable_list.cc index ee1563f01..dfa93461b 100644 --- a/db/memtable_list.cc +++ b/db/memtable_list.cc @@ -184,7 +184,7 @@ bool MemTableListVersion::GetFromList( assert(*seq != kMaxSequenceNumber || s->IsNotFound()); return true; } - if (!done && !s->ok() && !s->IsMergeInProgress() && !s->IsNotFound()) { + if (!s->ok() && !s->IsMergeInProgress() && !s->IsNotFound()) { return false; } } @@ -434,23 +434,57 @@ void MemTableList::PickMemtablesToFlush(uint64_t max_memtable_id, } void MemTableList::RollbackMemtableFlush(const autovector& mems, - uint64_t /*file_number*/) { + bool rollback_succeeding_memtables) { + TEST_SYNC_POINT("RollbackMemtableFlush"); AutoThreadOperationStageUpdater stage_updater( ThreadStatus::STAGE_MEMTABLE_ROLLBACK); - assert(!mems.empty()); - - // If the flush was not successful, then just reset state. - // Maybe a succeeding attempt to flush will be successful. +#ifndef NDEBUG for (MemTable* m : mems) { assert(m->flush_in_progress_); assert(m->file_number_ == 0); + } +#endif + + if (rollback_succeeding_memtables && !mems.empty()) { + std::list& memlist = current_->memlist_; + auto it = memlist.rbegin(); + for (; *it != mems[0] && it != memlist.rend(); ++it) { + } + // mems should be in memlist + assert(*it == mems[0]); + if (*it == mems[0]) { + ++it; + } + while (it != memlist.rend()) { + MemTable* m = *it; + // Only rollback complete, not in-progress, + // in_progress can be flushes that are still writing SSTs + if (m->flush_completed_) { + m->flush_in_progress_ = false; + m->flush_completed_ = false; + m->edit_.Clear(); + m->file_number_ = 0; + num_flush_not_started_++; + ++it; + } else { + break; + } + } + } - m->flush_in_progress_ = false; - m->flush_completed_ = false; - m->edit_.Clear(); - num_flush_not_started_++; + for (MemTable* m : mems) { + if (m->flush_in_progress_) { + assert(m->file_number_ == 0); + m->file_number_ = 0; + m->flush_in_progress_ = false; + m->flush_completed_ = false; + m->edit_.Clear(); + num_flush_not_started_++; + } + } + if (!mems.empty()) { + imm_flush_needed.store(true, std::memory_order_release); } - imm_flush_needed.store(true, std::memory_order_release); } // Try record a successful flush in the manifest file. It might just return diff --git a/db/memtable_list.h b/db/memtable_list.h index 1ad28a59e..81b60288d 100644 --- a/db/memtable_list.h +++ b/db/memtable_list.h @@ -271,8 +271,20 @@ class MemTableList { // Reset status of the given memtable list back to pending state so that // they can get picked up again on the next round of flush. + // + // @param rollback_succeeding_memtables If true, will rollback adjacent + // younger memtables whose flush is completed. Specifically, suppose the + // current immutable memtables are M_0,M_1...M_N ordered from youngest to + // oldest. Suppose that the youngest memtable in `mems` is M_K. We will try to + // rollback M_K-1, M_K-2... until the first memtable whose flush is + // not completed. These are the memtables that would have been installed + // by this flush job if it were to succeed. This flag is currently used + // by non atomic_flush rollback. + // Note that we also do rollback in `write_manifest_cb` by calling + // `RemoveMemTablesOrRestoreFlags()`. There we rollback the entire batch so + // it is similar to what we do here with rollback_succeeding_memtables=true. void RollbackMemtableFlush(const autovector& mems, - uint64_t file_number); + bool rollback_succeeding_memtables); // Try commit a successful flush in the manifest file. It might just return // Status::OK letting a concurrent flush to do the actual the recording. @@ -374,14 +386,43 @@ class MemTableList { return memlist.back()->GetID(); } - uint64_t GetLatestMemTableID() const { + uint64_t GetLatestMemTableID(bool for_atomic_flush) const { auto& memlist = current_->memlist_; if (memlist.empty()) { return 0; } + if (for_atomic_flush) { + // Scan the memtable list from new to old + for (auto it = memlist.begin(); it != memlist.end(); ++it) { + MemTable* m = *it; + if (m->atomic_flush_seqno_ != kMaxSequenceNumber) { + return m->GetID(); + } + } + return 0; + } return memlist.front()->GetID(); } + // DB mutex held. + // Gets the newest user-defined timestamp for the Memtables in ascending ID + // order, up to the `max_memtable_id`. Used by background flush job + // to check Memtables' eligibility for flush w.r.t retaining UDTs. + std::vector GetTablesNewestUDT(uint64_t max_memtable_id) { + std::vector newest_udts; + auto& memlist = current_->memlist_; + // Iterating through the memlist starting at the end, the vector + // ret is filled with memtables already sorted in increasing MemTable ID. + for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) { + MemTable* m = *it; + if (m->GetID() > max_memtable_id) { + break; + } + newest_udts.push_back(m->GetNewestUDT()); + } + return newest_udts; + } + void AssignAtomicFlushSeq(const SequenceNumber& seq) { const auto& memlist = current_->memlist_; // Scan the memtable list from new to old diff --git a/db/memtable_list_test.cc b/db/memtable_list_test.cc index c63952b12..9a5b7557f 100644 --- a/db/memtable_list_test.cc +++ b/db/memtable_list_test.cc @@ -43,6 +43,9 @@ class MemTableListTest : public testing::Test { // Open DB only with default column family ColumnFamilyOptions cf_options; std::vector cf_descs; + if (udt_enabled_) { + cf_options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + } cf_descs.emplace_back(kDefaultColumnFamilyName, cf_options); Status s = DB::Open(options, dbname, cf_descs, &handles, &db); EXPECT_OK(s); @@ -103,8 +106,9 @@ class MemTableListTest : public testing::Test { VersionSet versions(dbname, &immutable_db_options, env_options, table_cache.get(), &write_buffer_manager, &write_controller, /*block_cache_tracer=*/nullptr, - /*io_tracer=*/nullptr, /*db_id*/ "", - /*db_session_id*/ ""); + /*io_tracer=*/nullptr, /*db_id=*/"", + /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr); std::vector cf_descs; cf_descs.emplace_back(kDefaultColumnFamilyName, ColumnFamilyOptions()); cf_descs.emplace_back("one", ColumnFamilyOptions()); @@ -154,8 +158,9 @@ class MemTableListTest : public testing::Test { VersionSet versions(dbname, &immutable_db_options, env_options, table_cache.get(), &write_buffer_manager, &write_controller, /*block_cache_tracer=*/nullptr, - /*io_tracer=*/nullptr, /*db_id*/ "", - /*db_session_id*/ ""); + /*io_tracer=*/nullptr, /*db_id=*/"", + /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr); std::vector cf_descs; cf_descs.emplace_back(kDefaultColumnFamilyName, ColumnFamilyOptions()); cf_descs.emplace_back("one", ColumnFamilyOptions()); @@ -200,6 +205,9 @@ class MemTableListTest : public testing::Test { nullptr /* prep_tracker */, &mutex, file_meta_ptrs, committed_flush_jobs_info, to_delete, nullptr, &log_buffer); } + + protected: + bool udt_enabled_ = false; }; TEST_F(MemTableListTest, Empty) { @@ -676,7 +684,7 @@ TEST_F(MemTableListTest, FlushPendingTest) { ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); // Revert flush - list.RollbackMemtableFlush(to_flush, 0); + list.RollbackMemtableFlush(to_flush, false); ASSERT_FALSE(list.IsFlushPending()); ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); to_flush.clear(); @@ -726,7 +734,7 @@ TEST_F(MemTableListTest, FlushPendingTest) { ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); // Rollback first pick of tables - list.RollbackMemtableFlush(to_flush, 0); + list.RollbackMemtableFlush(to_flush, false); ASSERT_TRUE(list.IsFlushPending()); ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); to_flush.clear(); @@ -827,7 +835,7 @@ TEST_F(MemTableListTest, FlushPendingTest) { // Add another table list.Add(tables[5], &to_delete); ASSERT_EQ(1, list.NumNotFlushed()); - ASSERT_EQ(5, list.GetLatestMemTableID()); + ASSERT_EQ(5, list.GetLatestMemTableID(false /* for_atomic_flush */)); memtable_id = 4; // Pick tables to flush. The tables to pick must have ID smaller than or // equal to 4. Therefore, no table will be selected in this case. @@ -868,7 +876,7 @@ TEST_F(MemTableListTest, FlushPendingTest) { to_delete.clear(); } -TEST_F(MemTableListTest, EmptyAtomicFlusTest) { +TEST_F(MemTableListTest, EmptyAtomicFlushTest) { autovector lists; autovector cf_ids; autovector options_list; @@ -880,7 +888,7 @@ TEST_F(MemTableListTest, EmptyAtomicFlusTest) { ASSERT_TRUE(to_delete.empty()); } -TEST_F(MemTableListTest, AtomicFlusTest) { +TEST_F(MemTableListTest, AtomicFlushTest) { const int num_cfs = 3; const int num_tables_per_cf = 2; SequenceNumber seq = 1; @@ -1028,6 +1036,86 @@ TEST_F(MemTableListTest, AtomicFlusTest) { } } +class MemTableListWithTimestampTest : public MemTableListTest { + public: + MemTableListWithTimestampTest() : MemTableListTest() {} + + void SetUp() override { udt_enabled_ = true; } +}; + +TEST_F(MemTableListWithTimestampTest, GetTableNewestUDT) { + const int num_tables = 3; + const int num_entries = 5; + SequenceNumber seq = 1; + + auto factory = std::make_shared(); + options.memtable_factory = factory; + options.persist_user_defined_timestamps = false; + ImmutableOptions ioptions(options); + const Comparator* ucmp = test::BytewiseComparatorWithU64TsWrapper(); + InternalKeyComparator cmp(ucmp); + WriteBufferManager wb(options.db_write_buffer_size); + + // Create MemTableList + int min_write_buffer_number_to_merge = 1; + int max_write_buffer_number_to_maintain = 4; + int64_t max_write_buffer_size_to_maintain = + 4 * static_cast(options.write_buffer_size); + MemTableList list(min_write_buffer_number_to_merge, + max_write_buffer_number_to_maintain, + max_write_buffer_size_to_maintain); + + // Create some MemTables + uint64_t memtable_id = 0; + std::vector tables; + MutableCFOptions mutable_cf_options(options); + uint64_t current_ts = 0; + autovector to_delete; + std::vector newest_udts; + + std::string key; + std::string write_ts; + for (int i = 0; i < num_tables; i++) { + MemTable* mem = new MemTable(cmp, ioptions, mutable_cf_options, &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + mem->SetID(memtable_id++); + mem->Ref(); + + std::string value; + MergeContext merge_context; + + for (int j = 0; j < num_entries; j++) { + key = "key1"; + write_ts.clear(); + PutFixed64(&write_ts, current_ts); + key.append(write_ts); + ASSERT_OK(mem->Add(++seq, kTypeValue, key, std::to_string(i), + nullptr /* kv_prot_info */)); + current_ts++; + } + + tables.push_back(mem); + list.Add(tables.back(), &to_delete); + newest_udts.push_back(write_ts); + } + + ASSERT_EQ(num_tables, list.NumNotFlushed()); + ASSERT_TRUE(list.IsFlushPending()); + std::vector tables_newest_udts = list.GetTablesNewestUDT(num_tables); + ASSERT_EQ(newest_udts.size(), tables_newest_udts.size()); + for (size_t i = 0; i < tables_newest_udts.size(); i++) { + const Slice& table_newest_udt = tables_newest_udts[i]; + const Slice expected_newest_udt = newest_udts[i]; + ASSERT_EQ(expected_newest_udt, table_newest_udt); + } + + list.current()->Unref(&to_delete); + for (MemTable* m : to_delete) { + delete m; + } + to_delete.clear(); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/merge_helper.cc b/db/merge_helper.cc index 110ac9622..703909010 100644 --- a/db/merge_helper.cc +++ b/db/merge_helper.cc @@ -12,7 +12,7 @@ #include "db/blob/prefetch_buffer_collection.h" #include "db/compaction/compaction_iteration_stats.h" #include "db/dbformat.h" -#include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "logging/logging.h" #include "monitoring/perf_context_imp.h" #include "monitoring/statistics_impl.h" @@ -23,6 +23,7 @@ #include "rocksdb/system_clock.h" #include "table/format.h" #include "table/internal_iterator.h" +#include "util/overload.h" namespace ROCKSDB_NAMESPACE { @@ -56,120 +57,191 @@ MergeHelper::MergeHelper(Env* env, const Comparator* user_comparator, } } -Status MergeHelper::TimedFullMerge( - const MergeOperator* merge_operator, const Slice& key, const Slice* value, - const std::vector& operands, std::string* result, Logger* logger, - Statistics* statistics, SystemClock* clock, Slice* result_operand, - bool update_num_ops_stats, - MergeOperator::OpFailureScope* op_failure_scope) { - assert(merge_operator != nullptr); - - if (operands.empty()) { - assert(value != nullptr && result != nullptr); - result->assign(value->data(), value->size()); - return Status::OK(); - } +template +Status MergeHelper::TimedFullMergeCommonImpl( + const MergeOperator* merge_operator, const Slice& key, + MergeOperator::MergeOperationInputV3::ExistingValue&& existing_value, + const std::vector& operands, Logger* logger, Statistics* statistics, + SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, Visitor&& visitor) { + assert(merge_operator); + assert(!operands.empty()); if (update_num_ops_stats) { RecordInHistogram(statistics, READ_NUM_MERGE_OPERANDS, static_cast(operands.size())); } + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + MergeOperator::MergeOperationOutputV3 merge_out; + bool success = false; - Slice tmp_result_operand(nullptr, 0); - const MergeOperator::MergeOperationInput merge_in(key, value, operands, - logger); - MergeOperator::MergeOperationOutput merge_out(*result, tmp_result_operand); + { - // Setup to time the merge StopWatchNano timer(clock, statistics != nullptr); PERF_TIMER_GUARD(merge_operator_time_nanos); - // Do the merge - success = merge_operator->FullMergeV2(merge_in, &merge_out); - - if (tmp_result_operand.data()) { - // FullMergeV2 result is an existing operand - if (result_operand != nullptr) { - *result_operand = tmp_result_operand; - } else { - result->assign(tmp_result_operand.data(), tmp_result_operand.size()); - } - } else if (result_operand) { - *result_operand = Slice(nullptr, 0); - } + success = merge_operator->FullMergeV3(merge_in, &merge_out); RecordTick(statistics, MERGE_OPERATION_TOTAL_TIME, statistics ? timer.ElapsedNanos() : 0); } - if (op_failure_scope != nullptr) { - *op_failure_scope = merge_out.op_failure_scope; - // Apply default per merge_operator.h - if (*op_failure_scope == MergeOperator::OpFailureScope::kDefault) { - *op_failure_scope = MergeOperator::OpFailureScope::kTryMerge; - } - } - if (!success) { RecordTick(statistics, NUMBER_MERGE_FAILURES); + + if (op_failure_scope) { + *op_failure_scope = merge_out.op_failure_scope; + // Apply default per merge_operator.h + if (*op_failure_scope == MergeOperator::OpFailureScope::kDefault) { + *op_failure_scope = MergeOperator::OpFailureScope::kTryMerge; + } + } + return Status::Corruption(Status::SubCode::kMergeOperatorFailed); } - return Status::OK(); + return std::visit(std::forward(visitor), + std::move(merge_out.new_value)); } -Status MergeHelper::TimedFullMergeWithEntity( - const MergeOperator* merge_operator, const Slice& key, Slice base_entity, - const std::vector& operands, std::string* result, Logger* logger, - Statistics* statistics, SystemClock* clock, bool update_num_ops_stats, - MergeOperator::OpFailureScope* op_failure_scope) { - WideColumns base_columns; +Status MergeHelper::TimedFullMergeImpl( + const MergeOperator* merge_operator, const Slice& key, + MergeOperator::MergeOperationInputV3::ExistingValue&& existing_value, + const std::vector& operands, Logger* logger, Statistics* statistics, + SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, std::string* result, + Slice* result_operand, ValueType* result_type) { + assert(result); + assert(result_type); + + auto visitor = overload{ + [&](std::string&& new_value) -> Status { + *result_type = kTypeValue; + + if (result_operand) { + *result_operand = Slice(nullptr, 0); + } - { - const Status s = - WideColumnSerialization::Deserialize(base_entity, base_columns); - if (!s.ok()) { - return s; - } - } + *result = std::move(new_value); - const bool has_default_column = - !base_columns.empty() && base_columns[0].name() == kDefaultWideColumnName; + return Status::OK(); + }, + [&](MergeOperator::MergeOperationOutputV3::NewColumns&& new_columns) + -> Status { + *result_type = kTypeWideColumnEntity; - Slice value_of_default; - if (has_default_column) { - value_of_default = base_columns[0].value(); - } + if (result_operand) { + *result_operand = Slice(nullptr, 0); + } - std::string merge_result; + result->clear(); - { - const Status s = TimedFullMerge(merge_operator, key, &value_of_default, - operands, &merge_result, logger, statistics, - clock, nullptr /* result_operand */, - update_num_ops_stats, op_failure_scope); - if (!s.ok()) { - return s; - } - } + WideColumns sorted_columns; + sorted_columns.reserve(new_columns.size()); - if (has_default_column) { - base_columns[0].value() = merge_result; + for (const auto& column : new_columns) { + sorted_columns.emplace_back(column.first, column.second); + } - const Status s = WideColumnSerialization::Serialize(base_columns, *result); - if (!s.ok()) { - return s; - } - } else { - const Status s = - WideColumnSerialization::Serialize(merge_result, base_columns, *result); - if (!s.ok()) { - return s; - } - } + WideColumnsHelper::SortColumns(sorted_columns); - return Status::OK(); + return WideColumnSerialization::Serialize(sorted_columns, *result); + }, + [&](Slice&& operand) -> Status { + *result_type = kTypeValue; + + if (result_operand) { + *result_operand = operand; + result->clear(); + } else { + result->assign(operand.data(), operand.size()); + } + + return Status::OK(); + }}; + + return TimedFullMergeCommonImpl(merge_operator, key, + std::move(existing_value), operands, logger, + statistics, clock, update_num_ops_stats, + op_failure_scope, std::move(visitor)); +} + +Status MergeHelper::TimedFullMergeImpl( + const MergeOperator* merge_operator, const Slice& key, + MergeOperator::MergeOperationInputV3::ExistingValue&& existing_value, + const std::vector& operands, Logger* logger, Statistics* statistics, + SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, std::string* result_value, + PinnableWideColumns* result_entity) { + assert(result_value || result_entity); + assert(!result_value || !result_entity); + + auto visitor = overload{ + [&](std::string&& new_value) -> Status { + if (result_value) { + *result_value = std::move(new_value); + + return Status::OK(); + } + + assert(result_entity); + result_entity->SetPlainValue(std::move(new_value)); + + return Status::OK(); + }, + [&](MergeOperator::MergeOperationOutputV3::NewColumns&& new_columns) + -> Status { + if (result_value) { + if (!new_columns.empty() && + new_columns.front().first == kDefaultWideColumnName) { + *result_value = std::move(new_columns.front().second); + } else { + result_value->clear(); + } + + return Status::OK(); + } + + assert(result_entity); + + WideColumns sorted_columns; + sorted_columns.reserve(new_columns.size()); + + for (const auto& column : new_columns) { + sorted_columns.emplace_back(column.first, column.second); + } + + WideColumnsHelper::SortColumns(sorted_columns); + + std::string result; + const Status s = + WideColumnSerialization::Serialize(sorted_columns, result); + if (!s.ok()) { + result_entity->Reset(); + return s; + } + + return result_entity->SetWideColumnValue(std::move(result)); + }, + [&](Slice&& operand) -> Status { + if (result_value) { + result_value->assign(operand.data(), operand.size()); + + return Status::OK(); + } + + assert(result_entity); + result_entity->SetPlainValue(operand); + + return Status::OK(); + }}; + + return TimedFullMergeCommonImpl(merge_operator, key, + std::move(existing_value), operands, logger, + statistics, clock, update_num_ops_stats, + op_failure_scope, std::move(visitor)); } // PRE: iter points to the first merge type entry @@ -220,7 +292,9 @@ Status MergeHelper::MergeUntil(InternalIterator* iter, Status s = ParseInternalKey(original_key, &orig_ikey, allow_data_in_errors); assert(s.ok()); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } assert(kTypeMerge == orig_ikey.type); @@ -287,7 +361,7 @@ Status MergeHelper::MergeUntil(InternalIterator* iter, // hit a put/delete/single delete // => merge the put value or a nullptr with operands_ // => store result in operands_.back() (and update keys_.back()) - // => change the entry type to kTypeValue for keys_.back() + // => change the entry type for keys_.back() // We are done! Success! // If there are no operands, just return the Status::OK(). That will cause @@ -300,24 +374,23 @@ Status MergeHelper::MergeUntil(InternalIterator* iter, // TODO: if we're in compaction and it's a put, it would be nice to run // compaction filter on it. std::string merge_result; + ValueType merge_result_type; MergeOperator::OpFailureScope op_failure_scope; if (range_del_agg && range_del_agg->ShouldDelete( ikey, RangeDelPositioningMode::kForwardTraversal)) { - s = TimedFullMerge(user_merge_operator_, ikey.user_key, nullptr, - merge_context_.GetOperands(), &merge_result, logger_, - stats_, clock_, - /* result_operand */ nullptr, - /* update_num_ops_stats */ false, &op_failure_scope); + s = TimedFullMerge(user_merge_operator_, ikey.user_key, kNoBaseValue, + merge_context_.GetOperands(), logger_, stats_, + clock_, /* update_num_ops_stats */ false, + &op_failure_scope, &merge_result, + /* result_operand */ nullptr, &merge_result_type); } else if (ikey.type == kTypeValue) { - const Slice val = iter->value(); - - s = TimedFullMerge(user_merge_operator_, ikey.user_key, &val, - merge_context_.GetOperands(), &merge_result, logger_, - stats_, clock_, - /* result_operand */ nullptr, - /* update_num_ops_stats */ false, &op_failure_scope); + s = TimedFullMerge(user_merge_operator_, ikey.user_key, kPlainBaseValue, + iter->value(), merge_context_.GetOperands(), logger_, + stats_, clock_, /* update_num_ops_stats */ false, + &op_failure_scope, &merge_result, + /* result_operand */ nullptr, &merge_result_type); } else if (ikey.type == kTypeBlobIndex) { BlobIndex blob_index; @@ -347,22 +420,23 @@ Status MergeHelper::MergeUntil(InternalIterator* iter, c_iter_stats->total_blob_bytes_read += bytes_read; } - s = TimedFullMerge(user_merge_operator_, ikey.user_key, &blob_value, - merge_context_.GetOperands(), &merge_result, logger_, - stats_, clock_, - /* result_operand */ nullptr, - /* update_num_ops_stats */ false, &op_failure_scope); + s = TimedFullMerge(user_merge_operator_, ikey.user_key, kPlainBaseValue, + blob_value, merge_context_.GetOperands(), logger_, + stats_, clock_, /* update_num_ops_stats */ false, + &op_failure_scope, &merge_result, + /* result_operand */ nullptr, &merge_result_type); } else if (ikey.type == kTypeWideColumnEntity) { - s = TimedFullMergeWithEntity( - user_merge_operator_, ikey.user_key, iter->value(), - merge_context_.GetOperands(), &merge_result, logger_, stats_, - clock_, /* update_num_ops_stats */ false, &op_failure_scope); + s = TimedFullMerge(user_merge_operator_, ikey.user_key, kWideBaseValue, + iter->value(), merge_context_.GetOperands(), logger_, + stats_, clock_, /* update_num_ops_stats */ false, + &op_failure_scope, &merge_result, + /* result_operand */ nullptr, &merge_result_type); } else { - s = TimedFullMerge(user_merge_operator_, ikey.user_key, nullptr, - merge_context_.GetOperands(), &merge_result, logger_, - stats_, clock_, - /* result_operand */ nullptr, - /* update_num_ops_stats */ false, &op_failure_scope); + s = TimedFullMerge(user_merge_operator_, ikey.user_key, kNoBaseValue, + merge_context_.GetOperands(), logger_, stats_, + clock_, /* update_num_ops_stats */ false, + &op_failure_scope, &merge_result, + /* result_operand */ nullptr, &merge_result_type); } // We store the result in keys_.back() and operands_.back() @@ -370,10 +444,12 @@ Status MergeHelper::MergeUntil(InternalIterator* iter, if (s.ok()) { // The original key encountered original_key = std::move(keys_.back()); - orig_ikey.type = ikey.type == kTypeWideColumnEntity - ? kTypeWideColumnEntity - : kTypeValue; + + assert(merge_result_type == kTypeValue || + merge_result_type == kTypeWideColumnEntity); + orig_ikey.type = merge_result_type; UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type); + keys_.clear(); merge_context_.Clear(); keys_.emplace_front(std::move(original_key)); @@ -498,19 +574,24 @@ Status MergeHelper::MergeUntil(InternalIterator* iter, assert(merge_context_.GetNumOperands() >= 1); assert(merge_context_.GetNumOperands() == keys_.size()); std::string merge_result; + ValueType merge_result_type; MergeOperator::OpFailureScope op_failure_scope; - s = TimedFullMerge(user_merge_operator_, orig_ikey.user_key, nullptr, - merge_context_.GetOperands(), &merge_result, logger_, - stats_, clock_, - /* result_operand */ nullptr, - /* update_num_ops_stats */ false, &op_failure_scope); + s = TimedFullMerge(user_merge_operator_, orig_ikey.user_key, kNoBaseValue, + merge_context_.GetOperands(), logger_, stats_, clock_, + /* update_num_ops_stats */ false, &op_failure_scope, + &merge_result, + /* result_operand */ nullptr, &merge_result_type); if (s.ok()) { // The original key encountered // We are certain that keys_ is not empty here (see assertions couple of // lines before). original_key = std::move(keys_.back()); - orig_ikey.type = kTypeValue; + + assert(merge_result_type == kTypeValue || + merge_result_type == kTypeWideColumnEntity); + orig_ikey.type = merge_result_type; UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type); + keys_.clear(); merge_context_.Clear(); keys_.emplace_front(std::move(original_key)); diff --git a/db/merge_helper.h b/db/merge_helper.h index 7f624b743..39bd15f60 100644 --- a/db/merge_helper.h +++ b/db/merge_helper.h @@ -12,6 +12,7 @@ #include "db/merge_context.h" #include "db/range_del_aggregator.h" #include "db/snapshot_checker.h" +#include "db/wide/wide_column_serialization.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/env.h" #include "rocksdb/merge_operator.h" @@ -41,29 +42,92 @@ class MergeHelper { Statistics* stats = nullptr, const std::atomic* shutting_down = nullptr); - // Wrapper around MergeOperator::FullMergeV2() that records perf statistics. - // Result of merge will be written to result if status returned is OK. - // If operands is empty, the value will simply be copied to result. - // Set `update_num_ops_stats` to true if it is from a user read, so that - // the latency is sensitive. + // Wrappers around MergeOperator::FullMergeV3() that record perf statistics. + // Set `update_num_ops_stats` to true if it is from a user read so that + // the corresponding statistics are updated. // Returns one of the following statuses: // - OK: Entries were successfully merged. // - Corruption: Merge operator reported unsuccessful merge. The scope of the // damage will be stored in `*op_failure_scope` when `op_failure_scope` is // not nullptr + + // Empty tag types to disambiguate overloads + struct NoBaseValueTag {}; + static constexpr NoBaseValueTag kNoBaseValue{}; + + struct PlainBaseValueTag {}; + static constexpr PlainBaseValueTag kPlainBaseValue{}; + + struct WideBaseValueTag {}; + static constexpr WideBaseValueTag kWideBaseValue{}; + + template static Status TimedFullMerge(const MergeOperator* merge_operator, - const Slice& key, const Slice* value, + const Slice& key, NoBaseValueTag, const std::vector& operands, - std::string* result, Logger* logger, - Statistics* statistics, SystemClock* clock, - Slice* result_operand, bool update_num_ops_stats, - MergeOperator::OpFailureScope* op_failure_scope); - - static Status TimedFullMergeWithEntity( - const MergeOperator* merge_operator, const Slice& key, Slice base_entity, - const std::vector& operands, std::string* result, Logger* logger, + Logger* logger, Statistics* statistics, + SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, + ResultTs... results) { + MergeOperator::MergeOperationInputV3::ExistingValue existing_value; + + return TimedFullMergeImpl( + merge_operator, key, std::move(existing_value), operands, logger, + statistics, clock, update_num_ops_stats, op_failure_scope, results...); + } + + template + static Status TimedFullMerge( + const MergeOperator* merge_operator, const Slice& key, PlainBaseValueTag, + const Slice& value, const std::vector& operands, Logger* logger, Statistics* statistics, SystemClock* clock, bool update_num_ops_stats, - MergeOperator::OpFailureScope* op_failure_scope); + MergeOperator::OpFailureScope* op_failure_scope, ResultTs... results) { + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(value); + + return TimedFullMergeImpl( + merge_operator, key, std::move(existing_value), operands, logger, + statistics, clock, update_num_ops_stats, op_failure_scope, results...); + } + + template + static Status TimedFullMerge( + const MergeOperator* merge_operator, const Slice& key, WideBaseValueTag, + const Slice& entity, const std::vector& operands, Logger* logger, + Statistics* statistics, SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, ResultTs... results) { + MergeOperator::MergeOperationInputV3::ExistingValue existing_value; + + Slice entity_copy(entity); + WideColumns existing_columns; + + const Status s = + WideColumnSerialization::Deserialize(entity_copy, existing_columns); + if (!s.ok()) { + return s; + } + + existing_value = std::move(existing_columns); + + return TimedFullMergeImpl( + merge_operator, key, std::move(existing_value), operands, logger, + statistics, clock, update_num_ops_stats, op_failure_scope, results...); + } + + template + static Status TimedFullMerge(const MergeOperator* merge_operator, + const Slice& key, WideBaseValueTag, + const WideColumns& columns, + const std::vector& operands, + Logger* logger, Statistics* statistics, + SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, + ResultTs... results) { + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(columns); + + return TimedFullMergeImpl( + merge_operator, key, std::move(existing_value), operands, logger, + statistics, clock, update_num_ops_stats, op_failure_scope, results...); + } // During compaction, merge entries until we hit // - a corrupted key @@ -198,6 +262,36 @@ class MergeHelper { // This is a best-effort facility, so memory_order_relaxed is sufficient. return shutting_down_ && shutting_down_->load(std::memory_order_relaxed); } + + template + static Status TimedFullMergeCommonImpl( + const MergeOperator* merge_operator, const Slice& key, + MergeOperator::MergeOperationInputV3::ExistingValue&& existing_value, + const std::vector& operands, Logger* logger, + Statistics* statistics, SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, Visitor&& visitor); + + // Variant that exposes the merge result directly (in serialized form for wide + // columns) as well as its value type. Used by iterator and compaction. + static Status TimedFullMergeImpl( + const MergeOperator* merge_operator, const Slice& key, + MergeOperator::MergeOperationInputV3::ExistingValue&& existing_value, + const std::vector& operands, Logger* logger, + Statistics* statistics, SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, std::string* result, + Slice* result_operand, ValueType* result_type); + + // Variant that exposes the merge result translated into the form requested by + // the client. (For example, if the result is a wide-column structure but the + // client requested the results in plain-value form, the value of the default + // column is returned.) Used by point lookups. + static Status TimedFullMergeImpl( + const MergeOperator* merge_operator, const Slice& key, + MergeOperator::MergeOperationInputV3::ExistingValue&& existing_value, + const std::vector& operands, Logger* logger, + Statistics* statistics, SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope, + std::string* result_value, PinnableWideColumns* result_entity); }; // MergeOutputIterator can be used to iterate over the result of a merge. diff --git a/db/merge_operator.cc b/db/merge_operator.cc index d32585640..bb5dbbc36 100644 --- a/db/merge_operator.cc +++ b/db/merge_operator.cc @@ -9,6 +9,11 @@ #include "rocksdb/merge_operator.h" +#include + +#include "db/wide/wide_columns_helper.h" +#include "util/overload.h" + namespace ROCKSDB_NAMESPACE { bool MergeOperator::FullMergeV2(const MergeOperationInput& merge_in, @@ -23,6 +28,83 @@ bool MergeOperator::FullMergeV2(const MergeOperationInput& merge_in, &merge_out->new_value, merge_in.logger); } +bool MergeOperator::FullMergeV3(const MergeOperationInputV3& merge_in, + MergeOperationOutputV3* merge_out) const { + assert(merge_out); + + MergeOperationInput in_v2(merge_in.key, nullptr, merge_in.operand_list, + merge_in.logger); + + std::string new_value; + Slice existing_operand(nullptr, 0); + MergeOperationOutput out_v2(new_value, existing_operand); + + return std::visit( + overload{ + [&](const auto& existing) -> bool { + using T = std::decay_t; + + if constexpr (std::is_same_v) { + in_v2.existing_value = &existing; + } + + const bool result = FullMergeV2(in_v2, &out_v2); + if (!result) { + merge_out->op_failure_scope = out_v2.op_failure_scope; + return false; + } + + if (existing_operand.data()) { + merge_out->new_value = existing_operand; + } else { + merge_out->new_value = std::move(new_value); + } + + return true; + }, + [&](const WideColumns& existing_columns) -> bool { + const bool has_default_column = + WideColumnsHelper::HasDefaultColumn(existing_columns); + + Slice value_of_default; + if (has_default_column) { + value_of_default = existing_columns.front().value(); + } + + in_v2.existing_value = &value_of_default; + + const bool result = FullMergeV2(in_v2, &out_v2); + if (!result) { + merge_out->op_failure_scope = out_v2.op_failure_scope; + return false; + } + + merge_out->new_value = MergeOperationOutputV3::NewColumns(); + auto& new_columns = std::get( + merge_out->new_value); + new_columns.reserve(has_default_column + ? existing_columns.size() + : (existing_columns.size() + 1)); + + if (existing_operand.data()) { + new_columns.emplace_back(kDefaultWideColumnName.ToString(), + existing_operand.ToString()); + } else { + new_columns.emplace_back(kDefaultWideColumnName.ToString(), + std::move(new_value)); + } + + for (size_t i = has_default_column ? 1 : 0; + i < existing_columns.size(); ++i) { + new_columns.emplace_back(existing_columns[i].name().ToString(), + existing_columns[i].value().ToString()); + } + + return true; + }}, + merge_in.existing_value); +} + // The default implementation of PartialMergeMulti, which invokes // PartialMerge multiple times internally and merges two operands at // a time. diff --git a/db/merge_test.cc b/db/merge_test.cc index 6d1333e55..695ebe688 100644 --- a/db/merge_test.cc +++ b/db/merge_test.cc @@ -3,8 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). // -#include - +#include #include #include @@ -18,6 +17,7 @@ #include "rocksdb/env.h" #include "rocksdb/merge_operator.h" #include "rocksdb/utilities/db_ttl.h" +#include "rocksdb/wide_columns.h" #include "test_util/testharness.h" #include "util/coding.h" #include "utilities/merge_operators.h" @@ -86,7 +86,9 @@ class EnvMergeTest : public EnvWrapper { static std::unique_ptr singleton_; static EnvMergeTest* GetInstance() { - if (nullptr == singleton_) singleton_.reset(new EnvMergeTest); + if (nullptr == singleton_) { + singleton_.reset(new EnvMergeTest); + } return singleton_.get(); } }; @@ -144,7 +146,7 @@ class Counters { assert(db_); } - virtual ~Counters() {} + virtual ~Counters() = default; // public interface of Counters. // All four functions return false @@ -193,7 +195,7 @@ class Counters { std::cerr << "value corruption\n"; return false; } - *value = DecodeFixed64(&str[0]); + *value = DecodeFixed64(str.data()); return true; } else { std::cerr << s.ToString() << std::endl; @@ -219,14 +221,18 @@ class Counters { uint64_t value = default_; int result = get(key, &value); assert(result); - if (result == 0) exit(1); // Disable unused variable warning. + if (result == 0) { + exit(1); // Disable unused variable warning. + } return value; } void assert_add(const std::string& key, uint64_t value) { int result = add(key, value); assert(result); - if (result == 0) exit(1); // Disable unused variable warning. + if (result == 0) { + exit(1); // Disable unused variable warning. + } } }; @@ -495,7 +501,7 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges, std::string get_value_str; ASSERT_OK(db->Get(ReadOptions(), key, &get_value_str)); assert(get_value_str.size() == sizeof(uint64_t)); - uint64_t get_value = DecodeFixed64(&get_value_str[0]); + uint64_t get_value = DecodeFixed64(get_value_str.data()); ASSERT_EQ(get_value, num_merges * merge_value); ASSERT_EQ(num_merge_operator_calls, static_cast((num_merges % (max_num_merges + 1)))); @@ -607,6 +613,272 @@ TEST_F(MergeTest, MergeWithCompactionAndFlush) { ASSERT_OK(DestroyDB(dbname, Options())); } +TEST_F(MergeTest, FullMergeV3FallbackNewValue) { + // Test that the default FullMergeV3 implementation correctly handles the case + // when FullMergeV2 results in a new value. + + const Slice key("foo"); + const MergeOperator::MergeOperationInputV3::OperandList operands{ + "first", "second", "third"}; + constexpr Logger* logger = nullptr; + + auto append_operator = + MergeOperators::CreateStringAppendOperator(std::string()); + + // No existing value + { + MergeOperator::MergeOperationInputV3::ExistingValue existing_value; + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_TRUE(append_operator->FullMergeV3(merge_in, &merge_out)); + + const auto& result = std::get(merge_out.new_value); + ASSERT_EQ(result, operands[0].ToString() + operands[1].ToString() + + operands[2].ToString()); + } + + // Plain existing value + { + const Slice plain("plain"); + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(plain); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_TRUE(append_operator->FullMergeV3(merge_in, &merge_out)); + + const auto& result = std::get(merge_out.new_value); + ASSERT_EQ(result, plain.ToString() + operands[0].ToString() + + operands[1].ToString() + operands[2].ToString()); + } + + // Wide-column existing value with default column + { + const WideColumns entity{ + {kDefaultWideColumnName, "default"}, {"one", "1"}, {"two", "2"}}; + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(entity); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_TRUE(append_operator->FullMergeV3(merge_in, &merge_out)); + + const auto& result = + std::get( + merge_out.new_value); + ASSERT_EQ(result.size(), entity.size()); + ASSERT_EQ(result[0].first, entity[0].name()); + ASSERT_EQ(result[0].second, + entity[0].value().ToString() + operands[0].ToString() + + operands[1].ToString() + operands[2].ToString()); + ASSERT_EQ(result[1].first, entity[1].name()); + ASSERT_EQ(result[1].second, entity[1].value()); + ASSERT_EQ(result[2].first, entity[2].name()); + ASSERT_EQ(result[2].second, entity[2].value()); + } + + // Wide-column existing value without default column + { + const WideColumns entity{{"one", "1"}, {"two", "2"}}; + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(entity); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_TRUE(append_operator->FullMergeV3(merge_in, &merge_out)); + + const auto& result = + std::get( + merge_out.new_value); + ASSERT_EQ(result.size(), entity.size() + 1); + ASSERT_EQ(result[0].first, kDefaultWideColumnName); + ASSERT_EQ(result[0].second, operands[0].ToString() + + operands[1].ToString() + + operands[2].ToString()); + ASSERT_EQ(result[1].first, entity[0].name()); + ASSERT_EQ(result[1].second, entity[0].value()); + ASSERT_EQ(result[2].first, entity[1].name()); + ASSERT_EQ(result[2].second, entity[1].value()); + } +} + +TEST_F(MergeTest, FullMergeV3FallbackExistingOperand) { + // Test that the default FullMergeV3 implementation correctly handles the case + // when FullMergeV2 results in an existing operand. + + const Slice key("foo"); + const MergeOperator::MergeOperationInputV3::OperandList operands{ + "first", "second", "third"}; + constexpr Logger* logger = nullptr; + + auto put_operator = MergeOperators::CreatePutOperator(); + + // No existing value + { + MergeOperator::MergeOperationInputV3::ExistingValue existing_value; + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_TRUE(put_operator->FullMergeV3(merge_in, &merge_out)); + + const auto& result = std::get(merge_out.new_value); + ASSERT_EQ(result.data(), operands.back().data()); + ASSERT_EQ(result.size(), operands.back().size()); + } + + // Plain existing value + { + const Slice plain("plain"); + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(plain); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_TRUE(put_operator->FullMergeV3(merge_in, &merge_out)); + + const auto& result = std::get(merge_out.new_value); + ASSERT_EQ(result.data(), operands.back().data()); + ASSERT_EQ(result.size(), operands.back().size()); + } + + // Wide-column existing value with default column + { + const WideColumns entity{ + {kDefaultWideColumnName, "default"}, {"one", "1"}, {"two", "2"}}; + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(entity); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_TRUE(put_operator->FullMergeV3(merge_in, &merge_out)); + + const auto& result = + std::get( + merge_out.new_value); + ASSERT_EQ(result.size(), entity.size()); + ASSERT_EQ(result[0].first, entity[0].name()); + ASSERT_EQ(result[0].second, operands.back()); + ASSERT_EQ(result[1].first, entity[1].name()); + ASSERT_EQ(result[1].second, entity[1].value()); + ASSERT_EQ(result[2].first, entity[2].name()); + ASSERT_EQ(result[2].second, entity[2].value()); + } + + // Wide-column existing value without default column + { + const WideColumns entity{{"one", "1"}, {"two", "2"}}; + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(entity); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_TRUE(put_operator->FullMergeV3(merge_in, &merge_out)); + + const auto& result = + std::get( + merge_out.new_value); + ASSERT_EQ(result.size(), entity.size() + 1); + ASSERT_EQ(result[0].first, kDefaultWideColumnName); + ASSERT_EQ(result[0].second, operands.back()); + ASSERT_EQ(result[1].first, entity[0].name()); + ASSERT_EQ(result[1].second, entity[0].value()); + ASSERT_EQ(result[2].first, entity[1].name()); + ASSERT_EQ(result[2].second, entity[1].value()); + } +} + +TEST_F(MergeTest, FullMergeV3FallbackFailure) { + // Test that the default FullMergeV3 implementation correctly handles the case + // when FullMergeV2 fails. + + const Slice key("foo"); + const MergeOperator::MergeOperationInputV3::OperandList operands{ + "first", "second", "third"}; + constexpr Logger* logger = nullptr; + + class FailMergeOperator : public MergeOperator { + public: + bool FullMergeV2(const MergeOperationInput& /* merge_in */, + MergeOperationOutput* merge_out) const override { + assert(merge_out); + merge_out->op_failure_scope = OpFailureScope::kMustMerge; + + return false; + } + + const char* Name() const override { return "FailMergeOperator"; } + }; + + FailMergeOperator fail_operator; + + // No existing value + { + MergeOperator::MergeOperationInputV3::ExistingValue existing_value; + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_FALSE(fail_operator.FullMergeV3(merge_in, &merge_out)); + ASSERT_EQ(merge_out.op_failure_scope, + MergeOperator::OpFailureScope::kMustMerge); + } + + // Plain existing value + { + const Slice plain("plain"); + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(plain); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_FALSE(fail_operator.FullMergeV3(merge_in, &merge_out)); + ASSERT_EQ(merge_out.op_failure_scope, + MergeOperator::OpFailureScope::kMustMerge); + } + + // Wide-column existing value with default column + { + const WideColumns entity{ + {kDefaultWideColumnName, "default"}, {"one", "1"}, {"two", "2"}}; + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(entity); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_FALSE(fail_operator.FullMergeV3(merge_in, &merge_out)); + ASSERT_EQ(merge_out.op_failure_scope, + MergeOperator::OpFailureScope::kMustMerge); + } + + // Wide-column existing value without default column + { + const WideColumns entity{{"one", "1"}, {"two", "2"}}; + MergeOperator::MergeOperationInputV3::ExistingValue existing_value(entity); + const MergeOperator::MergeOperationInputV3 merge_in( + key, std::move(existing_value), operands, logger); + + MergeOperator::MergeOperationOutputV3 merge_out; + + ASSERT_FALSE(fail_operator.FullMergeV3(merge_in, &merge_out)); + ASSERT_EQ(merge_out.op_failure_scope, + MergeOperator::OpFailureScope::kMustMerge); + } +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/obsolete_files_test.cc b/db/obsolete_files_test.cc index 03f38c09f..d77594d60 100644 --- a/db/obsolete_files_test.cc +++ b/db/obsolete_files_test.cc @@ -7,10 +7,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include - #include +#include #include #include #include @@ -68,7 +66,7 @@ class ObsoleteFilesTest : public DBTestBase { int log_cnt = 0; int sst_cnt = 0; int manifest_cnt = 0; - for (auto file : filenames) { + for (const auto& file : filenames) { uint64_t number; FileType type; if (ParseFileName(file, &number, &type)) { @@ -165,7 +163,7 @@ TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) { {{"paranoid_file_checks", "true"}})); } } - ASSERT_OK(dbfull()->EnableFileDeletions(true /* force */)); + ASSERT_OK(dbfull()->EnableFileDeletions(/*force=*/false)); Close(); diff --git a/db/options_file_test.cc b/db/options_file_test.cc index c3adbeb64..7e48f0cf3 100644 --- a/db/options_file_test.cc +++ b/db/options_file_test.cc @@ -28,7 +28,7 @@ void UpdateOptionsFiles(DB* db, uint64_t number; FileType type; *options_files_count = 0; - for (auto filename : filenames) { + for (const auto& filename : filenames) { if (ParseFileName(filename, &number, &type) && type == kOptionsFile) { filename_history->insert(filename); (*options_files_count)++; @@ -44,16 +44,16 @@ void VerifyOptionsFileName( EXPECT_OK(db->GetEnv()->GetChildren(db->GetName(), &filenames)); uint64_t number; FileType type; - for (auto filename : filenames) { + for (const auto& filename : filenames) { if (ParseFileName(filename, &number, &type) && type == kOptionsFile) { current_filenames.insert(filename); } } - for (auto past_filename : past_filenames) { + for (const auto& past_filename : past_filenames) { if (current_filenames.find(past_filename) != current_filenames.end()) { continue; } - for (auto filename : current_filenames) { + for (const auto& filename : current_filenames) { ASSERT_GT(filename, past_filename); } } diff --git a/db/perf_context_test.cc b/db/perf_context_test.cc index bb8691b96..2666b8733 100644 --- a/db/perf_context_test.cc +++ b/db/perf_context_test.cc @@ -149,6 +149,7 @@ TEST_F(PerfContextTest, SeekIntoDeletion) { ASSERT_TRUE(iter->Valid()); StopWatchNano timer2(SystemClock::Default().get(), true); iter->Next(); + ASSERT_OK(iter->status()); auto elapsed_nanos2 = timer2.ElapsedNanos(); if (FLAGS_verbose) { std::cout << "next cmp: " << get_perf_context()->user_key_comparison_count @@ -262,7 +263,7 @@ void ProfileQueries(bool enabled_time = false) { for (const int i : keys) { if (i == kFlushFlag) { FlushOptions fo; - db->Flush(fo); + ASSERT_OK(db->Flush(fo)); continue; } @@ -1049,7 +1050,7 @@ TEST_F(PerfContextTest, MergeOperandCount) { std::vector statuses(num_keys); db->MultiGet(ReadOptions(), db->DefaultColumnFamily(), num_keys, - &key_slices[0], &results[0], &statuses[0]); + key_slices.data(), results.data(), statuses.data()); for (size_t i = 0; i < num_keys; ++i) { ASSERT_OK(statuses[i]); @@ -1067,7 +1068,7 @@ TEST_F(PerfContextTest, MergeOperandCount) { std::vector statuses(num_keys); db->MultiGetEntity(ReadOptions(), db->DefaultColumnFamily(), num_keys, - &key_slices[0], &results[0], &statuses[0]); + key_slices.data(), results.data(), statuses.data()); for (size_t i = 0; i < num_keys; ++i) { ASSERT_OK(statuses[i]); @@ -1092,6 +1093,7 @@ TEST_F(PerfContextTest, MergeOperandCount) { get_perf_context()->Reset(); } + ASSERT_OK(it->status()); } // Backward iteration @@ -1104,6 +1106,7 @@ TEST_F(PerfContextTest, MergeOperandCount) { get_perf_context()->Reset(); } + ASSERT_OK(it->status()); } }; @@ -1111,7 +1114,7 @@ TEST_F(PerfContextTest, MergeOperandCount) { verify(); // Verify counters when reading from table files - db->Flush(FlushOptions()); + ASSERT_OK(db->Flush(FlushOptions())); verify(); } diff --git a/db/periodic_task_scheduler.cc b/db/periodic_task_scheduler.cc index 1306f45da..1c4fc16b1 100644 --- a/db/periodic_task_scheduler.cc +++ b/db/periodic_task_scheduler.cc @@ -94,7 +94,7 @@ Status PeriodicTaskScheduler::Unregister(PeriodicTaskType task_type) { } Timer* PeriodicTaskScheduler::Default() { - static Timer timer(SystemClock::Default().get()); + STATIC_AVOID_DESTRUCTION(Timer, timer)(SystemClock::Default().get()); return &timer; } @@ -108,4 +108,3 @@ void PeriodicTaskScheduler::TEST_OverrideTimer(SystemClock* clock) { #endif // NDEBUG } // namespace ROCKSDB_NAMESPACE - diff --git a/db/periodic_task_scheduler.h b/db/periodic_task_scheduler.h index 4d129a679..a93f9a095 100644 --- a/db/periodic_task_scheduler.h +++ b/db/periodic_task_scheduler.h @@ -42,15 +42,16 @@ class PeriodicTaskScheduler { PeriodicTaskScheduler& operator=(const PeriodicTaskScheduler&) = delete; PeriodicTaskScheduler& operator=(PeriodicTaskScheduler&&) = delete; - // Register a task with its default repeat period + // Register a task with its default repeat period. Thread safe call. Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn); // Register a task with specified repeat period. 0 is an invalid argument - // (kInvalidPeriodSec). To stop the task, please use Unregister() specifically + // (kInvalidPeriodSec). To stop the task, please use Unregister(). + // Thread safe call. Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn, uint64_t repeat_period_seconds); - // Unregister the task + // Unregister the task. Thread safe call. Status Unregister(PeriodicTaskType task_type); #ifndef NDEBUG @@ -105,4 +106,3 @@ class PeriodicTaskScheduler { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/db/plain_table_db_test.cc b/db/plain_table_db_test.cc index d117639a4..1fa8d8f54 100644 --- a/db/plain_table_db_test.cc +++ b/db/plain_table_db_test.cc @@ -292,7 +292,7 @@ class TestPlainTableReader : public PlainTableReader { table_properties_ = std::move(props); } - ~TestPlainTableReader() override {} + ~TestPlainTableReader() override = default; private: bool MatchBloom(uint32_t hash) const override { @@ -897,6 +897,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeys) { } ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); delete iter; } @@ -945,6 +946,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) { } ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); delete iter; } diff --git a/db/prefix_test.cc b/db/prefix_test.cc index a8ae04035..b55956aa8 100644 --- a/db/prefix_test.cc +++ b/db/prefix_test.cc @@ -89,8 +89,12 @@ class TestKeyComparator : public Comparator { const TestKey* key_a = &kkey_a; const TestKey* key_b = &kkey_b; if (key_a->prefix != key_b->prefix) { - if (key_a->prefix < key_b->prefix) return -1; - if (key_a->prefix > key_b->prefix) return 1; + if (key_a->prefix < key_b->prefix) { + return -1; + } + if (key_a->prefix > key_b->prefix) { + return 1; + } } else { EXPECT_TRUE(key_a->prefix == key_b->prefix); // note, both a and b could be prefix only @@ -99,8 +103,12 @@ class TestKeyComparator : public Comparator { EXPECT_TRUE( (a.size() == sizeof(uint64_t) && b.size() == sizeof(TestKey)) || (b.size() == sizeof(uint64_t) && a.size() == sizeof(TestKey))); - if (a.size() < b.size()) return -1; - if (a.size() > b.size()) return 1; + if (a.size() < b.size()) { + return -1; + } + if (a.size() > b.size()) { + return 1; + } } else { // both a and b are prefix if (a.size() == sizeof(uint64_t)) { @@ -109,9 +117,15 @@ class TestKeyComparator : public Comparator { // both a and b are whole key EXPECT_TRUE(a.size() == sizeof(TestKey) && b.size() == sizeof(TestKey)); - if (key_a->sorted < key_b->sorted) return -1; - if (key_a->sorted > key_b->sorted) return 1; - if (key_a->sorted == key_b->sorted) return 0; + if (key_a->sorted < key_b->sorted) { + return -1; + } + if (key_a->sorted > key_b->sorted) { + return 1; + } + if (key_a->sorted == key_b->sorted) { + return 0; + } } } return 0; @@ -782,6 +796,7 @@ TEST_F(PrefixTest, PrefixSeekModePrev) { } } } + ASSERT_OK(iter->status()); } } @@ -891,4 +906,3 @@ int main(int argc, char** argv) { } #endif // GFLAGS - diff --git a/db/range_del_aggregator.cc b/db/range_del_aggregator.cc index 6e76f9c72..652afe65a 100644 --- a/db/range_del_aggregator.cc +++ b/db/range_del_aggregator.cc @@ -8,7 +8,6 @@ #include "db/compaction/compaction_iteration_stats.h" #include "db/dbformat.h" #include "db/pinned_iterators_manager.h" -#include "db/range_del_aggregator.h" #include "db/range_tombstone_fragmenter.h" #include "db/version_edit.h" #include "rocksdb/comparator.h" diff --git a/db/range_del_aggregator.h b/db/range_del_aggregator.h index dc1e73038..f7fa87af4 100644 --- a/db/range_del_aggregator.h +++ b/db/range_del_aggregator.h @@ -36,6 +36,10 @@ class TruncatedRangeDelIterator { const InternalKeyComparator* icmp, const InternalKey* smallest, const InternalKey* largest); + void SetRangeDelReadSeqno(SequenceNumber read_seqno) { + iter_->SetRangeDelReadSeqno(read_seqno); + } + bool Valid() const; void Next() { iter_->TopNext(); } diff --git a/db/range_tombstone_fragmenter.h b/db/range_tombstone_fragmenter.h index 8c7d98297..ce631d495 100644 --- a/db/range_tombstone_fragmenter.h +++ b/db/range_tombstone_fragmenter.h @@ -148,6 +148,10 @@ class FragmentedRangeTombstoneIterator : public InternalIterator { const InternalKeyComparator& icmp, SequenceNumber upper_bound, const Slice* ts_upper_bound = nullptr, SequenceNumber lower_bound = 0); + void SetRangeDelReadSeqno(SequenceNumber read_seqno) override { + upper_bound_ = read_seqno; + } + void SeekToFirst() override; void SeekToLast() override; diff --git a/db/repair.cc b/db/repair.cc index 58ada3aeb..ef21f7ea6 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -122,7 +122,8 @@ class Repairer { vset_(dbname_, &immutable_db_options_, file_options_, raw_table_cache_.get(), &wb_, &wc_, /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id=*/"", db_session_id_), + /*db_id=*/"", db_session_id_, db_options.daily_offpeak_time_utc, + /*error_handler=*/nullptr), next_file_number_(1), db_lock_(nullptr), closed_(false) { @@ -157,6 +158,7 @@ class Repairer { VersionEdit edit; edit.SetComparatorName(opts.comparator->Name()); + edit.SetPersistUserDefinedTimestamps(opts.persist_user_defined_timestamps); edit.SetLogNumber(0); edit.SetColumnFamily(cf_id); ColumnFamilyData* cfd; @@ -470,7 +472,7 @@ class Repairer { 0 /* file_creation_time */, "DB Repairer" /* db_id */, db_session_id_, 0 /*target_file_size*/, meta.fd.GetNumber()); - SeqnoToTimeMapping empty_seqno_time_mapping; + SeqnoToTimeMapping empty_seqno_to_time_mapping; status = BuildTable( dbname_, /* versions */ nullptr, immutable_db_options_, tboptions, file_options_, read_options, table_cache_.get(), iter.get(), @@ -478,8 +480,9 @@ class Repairer { {}, kMaxSequenceNumber, kMaxSequenceNumber, snapshot_checker, false /* paranoid_file_checks*/, nullptr /* internal_stats */, &io_s, nullptr /*IOTracer*/, BlobFileCreationReason::kRecovery, - empty_seqno_time_mapping, nullptr /* event_logger */, 0 /* job_id */, - Env::IO_HIGH, nullptr /* table_properties */, write_hint); + empty_seqno_to_time_mapping, nullptr /* event_logger */, + 0 /* job_id */, Env::IO_HIGH, nullptr /* table_properties */, + write_hint); ROCKS_LOG_INFO(db_options_.info_log, "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s", log, counter, meta.fd.GetNumber(), @@ -691,7 +694,9 @@ class Repairer { &cfd->internal_comparator(), cfd->user_comparator(), cfd->NumberLevels(), cfd->ioptions()->compaction_style, nullptr /* src_vstorage */, cfd->ioptions()->force_consistency_checks, - EpochNumberRequirement::kMightMissing); + EpochNumberRequirement::kMightMissing, cfd->ioptions()->clock, + /*bottommost_file_compaction_delay=*/0, + cfd->current()->version_set()->offpeak_time_option()); Status s; VersionEdit dummy_edit; for (const auto* table : cf_id_and_tables.second) { @@ -720,6 +725,8 @@ class Repairer { // recovered epoch numbers VersionEdit edit; edit.SetComparatorName(cfd->user_comparator()->Name()); + edit.SetPersistUserDefinedTimestamps( + cfd->ioptions()->persist_user_defined_timestamps); edit.SetLogNumber(0); edit.SetNextFile(next_file_number_); edit.SetColumnFamily(cfd->GetID()); diff --git a/db/repair_test.cc b/db/repair_test.cc index 8cca48424..8adc06f0c 100644 --- a/db/repair_test.cc +++ b/db/repair_test.cc @@ -365,18 +365,20 @@ TEST_P(RepairTestWithTimestamp, UnflushedSst) { Options options = CurrentOptions(); options.env = env_; options.create_if_missing = true; - std::string min_ts = Timestamp(0, 0); - std::string write_ts = Timestamp(1, 0); - const size_t kTimestampSize = write_ts.size(); - TestComparator test_cmp(kTimestampSize); - options.comparator = &test_cmp; + std::string min_ts; + std::string write_ts; + PutFixed64(&min_ts, 0); + PutFixed64(&write_ts, 1); + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); options.persist_user_defined_timestamps = persist_udt; + if (!persist_udt) { + options.allow_concurrent_memtable_write = false; + } options.paranoid_file_checks = paranoid_file_checks; ColumnFamilyOptions cf_options(options); std::vector column_families; - column_families.push_back( - ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + column_families.emplace_back(kDefaultColumnFamilyName, cf_options); ASSERT_OK(DB::Open(options, dbname_, column_families, &handles_, &db_)); diff --git a/db/seqno_time_test.cc b/db/seqno_time_test.cc index b18b25512..199c59c9b 100644 --- a/db/seqno_time_test.cc +++ b/db/seqno_time_test.cc @@ -12,13 +12,13 @@ #include "rocksdb/utilities/debug.h" #include "test_util/mock_time_env.h" - namespace ROCKSDB_NAMESPACE { class SeqnoTimeTest : public DBTestBase { public: SeqnoTimeTest() : DBTestBase("seqno_time_test", /*env_do_fsync=*/false) { mock_clock_ = std::make_shared(env_->GetSystemClock()); + mock_clock_->SetCurrentTime(kMockStartTime); mock_env_ = std::make_unique(env_, mock_clock_); } @@ -26,6 +26,10 @@ class SeqnoTimeTest : public DBTestBase { std::unique_ptr mock_env_; std::shared_ptr mock_clock_; + // Sufficient starting time that preserve time doesn't under-flow into + // pre-history + static constexpr uint32_t kMockStartTime = 10000000; + void SetUp() override { mock_clock_->InstallTimedWaitFixCallback(); SyncPoint::GetInstance()->SetCallBack( @@ -34,6 +38,7 @@ class SeqnoTimeTest : public DBTestBase { reinterpret_cast(arg); periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get()); }); + mock_clock_->SetCurrentTime(kMockStartTime); } // make sure the file is not in cache, otherwise it won't have IO info @@ -77,11 +82,6 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) { options.num_levels = kNumLevels; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun( - [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); - int sst_num = 0; // Write files that are overlap and enough to trigger compaction for (; sst_num < kNumTrigger; sst_num++) { @@ -189,11 +189,6 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) { options.disable_auto_compactions = true; DestroyAndReopen(options); - // pass some time first, otherwise the first a few keys write time are going - // to be zero, and internally zero has special meaning: kUnknownSeqnoTime - dbfull()->TEST_WaitForPeriodicTaskRun( - [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); - int sst_num = 0; // Write files that are overlap for (; sst_num < 4; sst_num++) { @@ -320,7 +315,9 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { DestroyAndReopen(options); std::set checked_file_nums; - SequenceNumber start_seq = dbfull()->GetLatestSequenceNumber(); + SequenceNumber start_seq = dbfull()->GetLatestSequenceNumber() + 1; + uint64_t start_time = mock_clock_->NowSeconds(); + // Write a key every 10 seconds for (int i = 0; i < 200; i++) { ASSERT_OK(Put(Key(i), "value")); @@ -338,21 +335,20 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { ASSERT_FALSE(tp_mapping.Empty()); auto seqs = tp_mapping.TEST_GetInternalMapping(); // about ~20 seqs->time entries, because the sample rate is 10000/100, and it - // passes 2k time. - ASSERT_GE(seqs.size(), 19); - ASSERT_LE(seqs.size(), 21); - SequenceNumber seq_end = dbfull()->GetLatestSequenceNumber(); - for (auto i = start_seq; i < start_seq + 10; i++) { - ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i + 1) * 10); - } - start_seq += 10; + // passes 2k time. Add (roughly) one for starting entry. + ASSERT_GE(seqs.size(), 20); + ASSERT_LE(seqs.size(), 22); + SequenceNumber seq_end = dbfull()->GetLatestSequenceNumber() + 1; for (auto i = start_seq; i < seq_end; i++) { // The result is within the range - ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), (i - 10) * 10); - ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i + 10) * 10); + ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 10 - 100); + ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 10); } checked_file_nums.insert(it->second->orig_file_number); start_seq = seq_end; + start_time = mock_clock_->NowSeconds(); // Write a key every 1 seconds for (int i = 0; i < 200; i++) { @@ -360,7 +356,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(1)); }); } - seq_end = dbfull()->GetLatestSequenceNumber(); + seq_end = dbfull()->GetLatestSequenceNumber() + 1; ASSERT_OK(Flush()); tables_props.clear(); ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); @@ -382,13 +378,14 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { ASSERT_GE(seqs.size(), 1); ASSERT_LE(seqs.size(), 3); for (auto i = start_seq; i < seq_end; i++) { - // The result is not very accurate, as there is more data write within small - // range of time - ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 1000); - ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 3000); + ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) - 100); + ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq)); } checked_file_nums.insert(it->second->orig_file_number); start_seq = seq_end; + start_time = mock_clock_->NowSeconds(); // Write a key every 200 seconds for (int i = 0; i < 200; i++) { @@ -396,7 +393,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(200)); }); } - seq_end = dbfull()->GetLatestSequenceNumber(); + seq_end = dbfull()->GetLatestSequenceNumber() + 1; ASSERT_OK(Flush()); tables_props.clear(); ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); @@ -417,20 +414,18 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { // The sequence number -> time entries should be maxed ASSERT_GE(seqs.size(), 99); ASSERT_LE(seqs.size(), 101); - for (auto i = start_seq; i < seq_end - 99; i++) { - // likely the first 100 entries reports 0 - ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 3000); - } - start_seq += 101; - for (auto i = start_seq; i < seq_end; i++) { - ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), - (i - start_seq) * 200 + 22200); - ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), - (i - start_seq) * 200 + 22600); + // aged out entries allowed to report time=0 + if ((seq_end - i) * 200 <= 10000) { + ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 200 - 100); + } + ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 200); } checked_file_nums.insert(it->second->orig_file_number); start_seq = seq_end; + start_time = mock_clock_->NowSeconds(); // Write a key every 100 seconds for (int i = 0; i < 200; i++) { @@ -438,7 +433,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } - seq_end = dbfull()->GetLatestSequenceNumber(); + seq_end = dbfull()->GetLatestSequenceNumber() + 1; ASSERT_OK(Flush()); tables_props.clear(); ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); @@ -484,18 +479,15 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { seqs = tp_mapping.TEST_GetInternalMapping(); ASSERT_GE(seqs.size(), 99); ASSERT_LE(seqs.size(), 101); - for (auto i = start_seq; i < seq_end - 99; i++) { - // likely the first 100 entries reports 0 - ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), - (i - start_seq) * 100 + 50000); - } - start_seq += 101; - for (auto i = start_seq; i < seq_end; i++) { - ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), - (i - start_seq) * 100 + 52200); - ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), - (i - start_seq) * 100 + 52400); + // aged out entries allowed to report time=0 + // FIXME: should be <= + if ((seq_end - i) * 100 < 10000) { + ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 100 - 100); + } + ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i), + start_time + (i - start_seq) * 100); } ASSERT_OK(db_->Close()); } @@ -620,14 +612,12 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) { ASSERT_GE(seqs.size(), 99); ASSERT_LE(seqs.size(), 101); - for (int j = 0; j < 2; j++) { for (int i = 0; i < 200; i++) { ASSERT_OK(Put(0, Key(i), "value")); dbfull()->TEST_WaitForPeriodicTaskRun( [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); } ASSERT_OK(Flush(0)); - } ASSERT_OK(dbfull()->TEST_WaitForCompact()); tables_props.clear(); ASSERT_OK(dbfull()->GetPropertiesOfAllTables(handles_[0], &tables_props)); @@ -735,8 +725,9 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) { ASSERT_OK(tp_mapping.Sort()); ASSERT_FALSE(tp_mapping.Empty()); auto seqs = tp_mapping.TEST_GetInternalMapping(); - ASSERT_GE(seqs.size(), 10 - 1); - ASSERT_LE(seqs.size(), 10 + 1); + // Add (roughly) one for starting entry. + ASSERT_GE(seqs.size(), 10); + ASSERT_LE(seqs.size(), 10 + 2); } // Trigger a compaction @@ -826,6 +817,203 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) { Close(); } +TEST_P(SeqnoTimeTablePropTest, PrePopulateInDB) { + Options base_options = CurrentOptions(); + base_options.env = mock_env_.get(); + base_options.disable_auto_compactions = true; + base_options.create_missing_column_families = true; + Options track_options = base_options; + constexpr uint32_t kPreserveSecs = 1234567; + SetTrackTimeDurationOptions(kPreserveSecs, track_options); + SeqnoToTimeMapping sttm; + SequenceNumber latest_seqno; + uint64_t start_time, end_time; + + // #### DB#1, #2: No pre-population without preserve/preclude #### + // #### But a single entry is added when preserve/preclude enabled #### + for (bool with_write : {false, true}) { + SCOPED_TRACE("with_write=" + std::to_string(with_write)); + DestroyAndReopen(base_options); + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + ASSERT_TRUE(sttm.Empty()); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + + if (with_write) { + // Ensure that writes before new CF with preserve/preclude option don't + // interfere with the seqno-to-time mapping getting a starting entry. + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + } else { + // FIXME: currently, starting entry after CreateColumnFamily requires + // non-zero seqno + ASSERT_OK(Delete("blah")); + } + + // Unfortunately, if we add a CF with preserve/preclude option after + // open, that does not reserve seqnos with pre-populated time mappings. + CreateColumnFamilies({"one"}, track_options); + + // No pre-population (unfortunately), just a single starting entry + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + latest_seqno = db_->GetLatestSequenceNumber(); + start_time = mock_clock_->NowSeconds(); + ASSERT_EQ(sttm.Size(), 1); + ASSERT_EQ(latest_seqno, 1U); + // Current time maps to starting entry / seqno + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time), 1U); + // Any older times are unknown. + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - 1), + kUnknownSeqnoBeforeAll); + + // Now check that writes can proceed normally (passing about 20% of preserve + // time) + for (int i = 0; i < 20; i++) { + ASSERT_OK(Put(Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kPreserveSecs / 99)); + }); + } + ASSERT_OK(Flush()); + + // Check that mappings are getting populated + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + latest_seqno = db_->GetLatestSequenceNumber(); + end_time = mock_clock_->NowSeconds(); + ASSERT_EQ(sttm.Size(), 21); + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(end_time), latest_seqno); + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time), 1U); + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - 1), + kUnknownSeqnoBeforeAll); + } + + // ### DB#3, #4: Read-only DB with preserve/preclude after not #### + // Make sure we don't hit issues with read-only DBs, which don't need + // the mapping in the DB state (though it wouldn't hurt anything) + for (bool with_write : {false, true}) { + SCOPED_TRACE("with_write=" + std::to_string(with_write)); + DestroyAndReopen(base_options); + if (with_write) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + } + + ASSERT_OK(ReadOnlyReopen(base_options)); + if (with_write) { + ASSERT_EQ(Get("foo"), "bar"); + } + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + ASSERT_EQ(sttm.Size(), 0); + if (!with_write) { + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0); + } + + ASSERT_OK(ReadOnlyReopen(track_options)); + if (with_write) { + ASSERT_EQ(Get("foo"), "bar"); + } + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + ASSERT_EQ(sttm.Size(), 0); + if (!with_write) { + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0); + + // And even if we re-open read-write, we do not get pre-population, + // because that's only for new DBs. + Reopen(track_options); + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + ASSERT_EQ(sttm.Size(), 0); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0); + } + } + + // #### DB#5: Destroy and open with preserve/preclude option #### + DestroyAndReopen(track_options); + + // Ensure pre-population + constexpr auto kPrePopPairs = SeqnoToTimeMapping::kMaxSeqnoTimePairsPerSST; + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + latest_seqno = db_->GetLatestSequenceNumber(); + start_time = mock_clock_->NowSeconds(); + ASSERT_EQ(sttm.Size(), kPrePopPairs); + // One nono-zero sequence number per pre-populated pair (this could be + // revised if we want to use interpolation for better approximate time + // mappings with no guarantee of erring in just one direction). + ASSERT_EQ(latest_seqno, kPrePopPairs); + // Current time maps to last pre-allocated seqno + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time), latest_seqno); + // Oldest tracking time maps to first pre-allocated seqno + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - kPreserveSecs), 1); + + // In more detail, check that estimated seqnos (pre-allocated) are uniformly + // spread over the tracked time. + for (auto ratio : {0.0, 0.433, 0.678, 0.987, 1.0}) { + // Round up query time + uint64_t t = start_time - kPreserveSecs + + static_cast(ratio * kPreserveSecs + 0.9999999); + // Round down estimated seqno + SequenceNumber s = + static_cast(ratio * (latest_seqno - 1)) + 1; + // Match + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(t), s); + } + + // Now check that writes can proceed normally (passing about 20% of preserve + // time) + for (int i = 0; i < 20; i++) { + ASSERT_OK(Put(Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kPreserveSecs / 99)); + }); + } + ASSERT_OK(Flush()); + + // Can still see some pre-populated mappings, though some displaced + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + latest_seqno = db_->GetLatestSequenceNumber(); + end_time = mock_clock_->NowSeconds(); + ASSERT_EQ(sttm.Size(), kPrePopPairs); + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(end_time), latest_seqno); + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - kPreserveSecs / 2), + kPrePopPairs / 2); + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - kPreserveSecs), + kUnknownSeqnoBeforeAll); + + // Make sure we don't hit issues with read-only DBs, which don't need + // the mapping in the DB state (though it wouldn't hurt anything) + ASSERT_OK(ReadOnlyReopen(track_options)); + ASSERT_EQ(Get(Key(0)), "value"); + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + ASSERT_EQ(sttm.Size(), 0); + + // #### DB#6: Destroy and open+create an extra CF with preserve/preclude #### + // (default CF does not have the option) + Destroy(track_options); + ReopenWithColumnFamilies({"default", "one"}, + List({base_options, track_options})); + + // Ensure pre-population (not as exhaustive checking here) + sttm = dbfull()->TEST_GetSeqnoToTimeMapping(); + latest_seqno = db_->GetLatestSequenceNumber(); + start_time = mock_clock_->NowSeconds(); + ASSERT_EQ(sttm.Size(), kPrePopPairs); + // One nono-zero sequence number per pre-populated pair (this could be + // revised if we want to use interpolation for better approximate time + // mappings with no guarantee of erring in just one direction). + ASSERT_EQ(latest_seqno, kPrePopPairs); + // Current time maps to last pre-allocated seqno + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time), latest_seqno); + // Oldest tracking time maps to first pre-allocated seqno + ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - kPreserveSecs), 1); + + // Even after no writes and DB re-open without tracking options, sequence + // numbers should not go backward into those that were pre-allocated. + // (Future work: persist the mapping) + ReopenWithColumnFamilies({"default", "one"}, + List({base_options, base_options})); + ASSERT_EQ(latest_seqno, db_->GetLatestSequenceNumber()); + + Close(); +} + TEST_F(SeqnoTimeTest, MappingAppend) { SeqnoToTimeMapping test(/*max_time_duration=*/100, /*max_capacity=*/10); @@ -843,8 +1031,9 @@ TEST_F(SeqnoTimeTest, MappingAppend) { ASSERT_FALSE(test.Append(8, 12)); ASSERT_EQ(size, test.Size()); - // Append with the same seqno, newer time will be accepted - ASSERT_TRUE(test.Append(10, 12)); + // Append with the same seqno, newer time is rejected because that makes + // GetProximalSeqnoBeforeTime queries worse (see later test) + ASSERT_FALSE(test.Append(10, 12)); ASSERT_EQ(size, test.Size()); // older time will be ignored ASSERT_FALSE(test.Append(10, 9)); @@ -853,25 +1042,220 @@ TEST_F(SeqnoTimeTest, MappingAppend) { // new seqno with old time will be ignored ASSERT_FALSE(test.Append(12, 8)); ASSERT_EQ(size, test.Size()); + + // new seqno with same time is accepted by replacing last entry + // (improves GetProximalSeqnoBeforeTime queries without blowing up size) + ASSERT_TRUE(test.Append(12, 11)); + ASSERT_EQ(size, test.Size()); } -TEST_F(SeqnoTimeTest, GetOldestApproximateTime) { +TEST_F(SeqnoTimeTest, ProximalFunctions) { SeqnoToTimeMapping test(/*max_time_duration=*/100, /*max_capacity=*/10); - ASSERT_EQ(test.GetOldestApproximateTime(10), kUnknownSeqnoTime); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(1), kUnknownTimeBeforeAll); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(1000000000000U), + kUnknownTimeBeforeAll); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(1), kUnknownSeqnoBeforeAll); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(1000000000000U), + kUnknownSeqnoBeforeAll); + + // (Taken from example in SeqnoToTimeMapping class comment) + // Time 500 is after seqno 10 and before seqno 11 + EXPECT_TRUE(test.Append(10, 500)); + + // Seqno too early + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(9), kUnknownTimeBeforeAll); + // We only know that 500 is after 10 + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(10), kUnknownTimeBeforeAll); + // Found + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(11), 500U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(1000000000000U), 500U); + + // Time too early + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(499), kUnknownSeqnoBeforeAll); + // Found + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(500), 10U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(501), 10U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(1000000000000U), 10U); + + // More samples + EXPECT_TRUE(test.Append(20, 600)); + EXPECT_TRUE(test.Append(30, 700)); + + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(10), kUnknownTimeBeforeAll); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(11), 500U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(20), 500U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(21), 600U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(30), 600U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(31), 700U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(1000000000000U), 700U); + + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(499), kUnknownSeqnoBeforeAll); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(500), 10U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(501), 10U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(599), 10U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(600), 20U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(601), 20U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(701), 30U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(1000000000000U), 30U); + + // Redundant sample ignored + EXPECT_EQ(test.Size(), 3U); + EXPECT_FALSE(test.Append(30, 700)); + EXPECT_EQ(test.Size(), 3U); + + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(30), 600U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(31), 700U); + + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U); + + // Later sample with same seqno is ignored, to provide best results + // for GetProximalSeqnoBeforeTime function while saving entries + // in SeqnoToTimeMapping. + EXPECT_FALSE(test.Append(30, 800)); + + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(30), 600U); + // Could return 800, but saving space in SeqnoToTimeMapping instead. + // Can reconsider if/when GetProximalTimeBeforeSeqno is used in + // production. + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(31), 700U); + + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U); + // If the existing {30, 700} entry were replaced with {30, 800}, this + // would return seqno 20 instead of 30, which would preclude more than + // necessary for "preclude_last_level_data_seconds" feature. + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(800), 30U); + + // Still OK + EXPECT_TRUE(test.Append(40, 900)); + + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(30), 600U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(41), 900U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(899), 30U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(900), 40U); + + // Burst of writes during a short time creates an opportunity + // for better results from GetProximalSeqnoBeforeTime(), at the + // expense of GetProximalTimeBeforeSeqno(). + EXPECT_TRUE(test.Append(50, 900)); + + // These are subject to later revision depending on priorities + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(49), 700U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(51), 900U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(899), 30U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(900), 50U); +} - test.Append(3, 10); +TEST_F(SeqnoTimeTest, PrePopulate) { + SeqnoToTimeMapping test(/*max_time_duration=*/100, /*max_capacity=*/10); - ASSERT_EQ(test.GetOldestApproximateTime(2), kUnknownSeqnoTime); - ASSERT_EQ(test.GetOldestApproximateTime(3), 10); - ASSERT_EQ(test.GetOldestApproximateTime(10), 10); + EXPECT_EQ(test.Size(), 0U); - test.Append(10, 100); + // Smallest case is like two Appends + test.PrePopulate(10, 11, 500, 600); + + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(10), kUnknownTimeBeforeAll); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(11), 500U); + EXPECT_EQ(test.GetProximalTimeBeforeSeqno(12), 600U); + + test.Clear(); + + // Populate a small range + uint64_t kTimeIncrement = 1234567; + test.PrePopulate(1, 12, kTimeIncrement, kTimeIncrement * 2); + + for (uint64_t i = 0; i <= 12; ++i) { + // NOTE: with 1 and 12 as the pre-populated end points, the duration is + // broken into 11 equal(-ish) spans + uint64_t t = kTimeIncrement + (i * kTimeIncrement) / 11 - 1; + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(t), i); + } + + test.Clear(); + + // Populate an excessively large range (in the future we might want to + // interpolate estimated times for seqnos between entries) + test.PrePopulate(1, 34567, kTimeIncrement, kTimeIncrement * 2); + + for (auto ratio : {0.0, 0.433, 0.678, 0.987, 1.0}) { + // Round up query time + uint64_t t = kTimeIncrement + + static_cast(ratio * kTimeIncrement + 0.9999999); + // Round down estimated seqno + SequenceNumber s = static_cast(ratio * (34567 - 1)) + 1; + // Match + // TODO: for now this is exact, but in the future might need approximation + // bounds to account for limited samples. + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(t), s); + } +} + +TEST_F(SeqnoTimeTest, TruncateOldEntries) { + constexpr uint64_t kMaxTimeDuration = 42; + SeqnoToTimeMapping test(kMaxTimeDuration, /*max_capacity=*/10); + + EXPECT_EQ(test.Size(), 0U); + + // Safe on empty mapping + test.TruncateOldEntries(500); + + EXPECT_EQ(test.Size(), 0U); + + // (Taken from example in SeqnoToTimeMapping class comment) + // Time 500 is after seqno 10 and before seqno 11 + EXPECT_TRUE(test.Append(10, 500)); + EXPECT_TRUE(test.Append(20, 600)); + EXPECT_TRUE(test.Append(30, 700)); + EXPECT_TRUE(test.Append(40, 800)); + EXPECT_TRUE(test.Append(50, 900)); + + EXPECT_EQ(test.Size(), 5U); + + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(500), 10U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(599), 10U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(600), 20U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U); + // etc. + + // Must keep first entry + test.TruncateOldEntries(500 + kMaxTimeDuration); + EXPECT_EQ(test.Size(), 5U); + test.TruncateOldEntries(599 + kMaxTimeDuration); + EXPECT_EQ(test.Size(), 5U); + + // Purges first entry + test.TruncateOldEntries(600 + kMaxTimeDuration); + EXPECT_EQ(test.Size(), 4U); + + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(500), kUnknownSeqnoBeforeAll); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(599), kUnknownSeqnoBeforeAll); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(600), 20U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U); + + // No effect + test.TruncateOldEntries(600 + kMaxTimeDuration); + EXPECT_EQ(test.Size(), 4U); + test.TruncateOldEntries(699 + kMaxTimeDuration); + EXPECT_EQ(test.Size(), 4U); + + // Purges next two + test.TruncateOldEntries(899 + kMaxTimeDuration); + EXPECT_EQ(test.Size(), 2U); + + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(799), kUnknownSeqnoBeforeAll); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(899), 40U); + + // Always keep last entry, to have a non-trivial seqno bound + test.TruncateOldEntries(10000000); + EXPECT_EQ(test.Size(), 1U); - test.Append(100, 1000); - ASSERT_EQ(test.GetOldestApproximateTime(10), 100); - ASSERT_EQ(test.GetOldestApproximateTime(40), 100); - ASSERT_EQ(test.GetOldestApproximateTime(111), 1000); + EXPECT_EQ(test.GetProximalSeqnoBeforeTime(10000000), 50U); } TEST_F(SeqnoTimeTest, Sort) { @@ -930,10 +1314,10 @@ TEST_F(SeqnoTimeTest, EncodeDecodeBasic) { for (SequenceNumber seq = 0; seq <= 1000; seq++) { // test has the more accurate time mapping, encode only pick // kMaxSeqnoTimePairsPerSST number of entries, which is less accurate - uint64_t target_time = test.GetOldestApproximateTime(seq); - ASSERT_GE(decoded.GetOldestApproximateTime(seq), + uint64_t target_time = test.GetProximalTimeBeforeSeqno(seq); + ASSERT_GE(decoded.GetProximalTimeBeforeSeqno(seq), target_time < 200 ? 0 : target_time - 200); - ASSERT_LE(decoded.GetOldestApproximateTime(seq), target_time); + ASSERT_LE(decoded.GetProximalTimeBeforeSeqno(seq), target_time); } } diff --git a/db/seqno_to_time_mapping.cc b/db/seqno_to_time_mapping.cc index c69209929..97a3e9879 100644 --- a/db/seqno_to_time_mapping.cc +++ b/db/seqno_to_time_mapping.cc @@ -11,14 +11,34 @@ namespace ROCKSDB_NAMESPACE { -uint64_t SeqnoToTimeMapping::GetOldestApproximateTime( - const SequenceNumber seqno) const { +SeqnoToTimeMapping::pair_const_iterator SeqnoToTimeMapping::FindGreaterTime( + uint64_t time) const { + return std::upper_bound(pairs_.cbegin(), pairs_.cend(), + SeqnoTimePair{0, time}, SeqnoTimePair::TimeLess); +} + +SeqnoToTimeMapping::pair_const_iterator SeqnoToTimeMapping::FindGreaterEqSeqno( + SequenceNumber seqno) const { + return std::lower_bound(pairs_.cbegin(), pairs_.cend(), + SeqnoTimePair{seqno, 0}, SeqnoTimePair::SeqnoLess); +} + +SeqnoToTimeMapping::pair_const_iterator SeqnoToTimeMapping::FindGreaterSeqno( + SequenceNumber seqno) const { + return std::upper_bound(pairs_.cbegin(), pairs_.cend(), + SeqnoTimePair{seqno, 0}, SeqnoTimePair::SeqnoLess); +} + +uint64_t SeqnoToTimeMapping::GetProximalTimeBeforeSeqno( + SequenceNumber seqno) const { assert(is_sorted_); - auto it = std::upper_bound(seqno_time_mapping_.begin(), - seqno_time_mapping_.end(), seqno); - if (it == seqno_time_mapping_.begin()) { - return 0; + // Find the last entry with a seqno strictly less than the given seqno. + // First, find the first entry >= the given seqno (or end) + auto it = FindGreaterEqSeqno(seqno); + if (it == pairs_.cbegin()) { + return kUnknownTimeBeforeAll; } + // Then return data from previous. it--; return it->time; } @@ -28,44 +48,47 @@ void SeqnoToTimeMapping::Add(SequenceNumber seqno, uint64_t time) { return; } is_sorted_ = false; - seqno_time_mapping_.emplace_back(seqno, time); + pairs_.emplace_back(seqno, time); } void SeqnoToTimeMapping::TruncateOldEntries(const uint64_t now) { assert(is_sorted_); if (max_time_duration_ == 0) { + // No cutoff time + return; + } + + if (now < max_time_duration_) { + // Would under-flow return; } - const uint64_t cut_off_time = - now > max_time_duration_ ? now - max_time_duration_ : 0; - assert(cut_off_time <= now); // no overflow + const uint64_t cut_off_time = now - max_time_duration_; + assert(cut_off_time <= now); // no under/overflow - auto it = std::upper_bound( - seqno_time_mapping_.begin(), seqno_time_mapping_.end(), cut_off_time, - [](uint64_t target, const SeqnoTimePair& other) -> bool { - return target < other.time; - }); - if (it == seqno_time_mapping_.begin()) { + auto it = FindGreaterTime(cut_off_time); + if (it == pairs_.cbegin()) { return; } - it--; - seqno_time_mapping_.erase(seqno_time_mapping_.begin(), it); + // Move back one, to the entry that would be used to return a good seqno from + // GetProximalSeqnoBeforeTime(cut_off_time) + --it; + // Remove everything strictly before that entry + pairs_.erase(pairs_.cbegin(), std::move(it)); } -SequenceNumber SeqnoToTimeMapping::GetOldestSequenceNum(uint64_t time) { +SequenceNumber SeqnoToTimeMapping::GetProximalSeqnoBeforeTime(uint64_t time) { assert(is_sorted_); - auto it = std::upper_bound( - seqno_time_mapping_.begin(), seqno_time_mapping_.end(), time, - [](uint64_t target, const SeqnoTimePair& other) -> bool { - return target < other.time; - }); - if (it == seqno_time_mapping_.begin()) { - return 0; + // Find the last entry with a time <= the given time. + // First, find the first entry > the given time (or end). + auto it = FindGreaterTime(time); + if (it == pairs_.cbegin()) { + return kUnknownSeqnoBeforeAll; } - it--; + // Then return data from previous. + --it; return it->seqno; } @@ -84,15 +107,13 @@ void SeqnoToTimeMapping::Encode(std::string& dest, const SequenceNumber start, return; } - auto start_it = std::upper_bound(seqno_time_mapping_.begin(), - seqno_time_mapping_.end(), start); - if (start_it != seqno_time_mapping_.begin()) { + auto start_it = FindGreaterSeqno(start); + if (start_it != pairs_.begin()) { start_it--; } - auto end_it = std::upper_bound(seqno_time_mapping_.begin(), - seqno_time_mapping_.end(), end); - if (end_it == seqno_time_mapping_.begin()) { + auto end_it = FindGreaterSeqno(end); + if (end_it == pairs_.begin()) { return; } if (start_it >= end_it) { @@ -108,7 +129,7 @@ void SeqnoToTimeMapping::Encode(std::string& dest, const SequenceNumber start, } } // to include the first element - if (start_it != seqno_time_mapping_.begin()) { + if (start_it != pairs_.begin()) { start_it--; } @@ -166,14 +187,14 @@ void SeqnoToTimeMapping::Encode(std::string& dest, const SequenceNumber start, SeqnoTimePair base; for (auto it = start_it; it < end_it; it++) { assert(base < *it); - SeqnoTimePair val = *it - base; + SeqnoTimePair val = it->ComputeDelta(base); base = *it; val.Encode(dest); } } -Status SeqnoToTimeMapping::Add(const std::string& seqno_time_mapping_str) { - Slice input(seqno_time_mapping_str); +Status SeqnoToTimeMapping::Add(const std::string& pairs_str) { + Slice input(pairs_str); if (input.empty()) { return Status::OK(); } @@ -189,8 +210,8 @@ Status SeqnoToTimeMapping::Add(const std::string& seqno_time_mapping_str) { if (!s.ok()) { return s; } - val.Add(base); - seqno_time_mapping_.emplace_back(val); + val.ApplyDelta(base); + pairs_.emplace_back(val); base = val; } return Status::OK(); @@ -222,33 +243,58 @@ bool SeqnoToTimeMapping::Append(SequenceNumber seqno, uint64_t time) { return false; } if (seqno == Last().seqno) { - Last().time = time; - return true; + // Updating Last() would hurt GetProximalSeqnoBeforeTime() queries, so + // NOT doing it (for now) + return false; } if (time == Last().time) { - // new sequence has the same time as old one, no need to add new mapping - return false; + // Updating Last() here helps GetProximalSeqnoBeforeTime() queries, so + // doing it (for now) + Last().seqno = seqno; + return true; } } - seqno_time_mapping_.emplace_back(seqno, time); + pairs_.emplace_back(seqno, time); - if (seqno_time_mapping_.size() > max_capacity_) { - seqno_time_mapping_.pop_front(); + if (pairs_.size() > max_capacity_) { + // FIXME: be smarter about how we erase to avoid data falling off the + // front prematurely. + pairs_.pop_front(); } return true; } +bool SeqnoToTimeMapping::PrePopulate(SequenceNumber from_seqno, + SequenceNumber to_seqno, + uint64_t from_time, uint64_t to_time) { + assert(Empty()); + assert(from_seqno > 0); + assert(to_seqno > from_seqno); + assert(from_time > kUnknownTimeBeforeAll); + assert(to_time >= from_time); + + // TODO: smartly limit this to max_capacity_ representative samples + for (auto i = from_seqno; i <= to_seqno; i++) { + uint64_t t = from_time + (to_time - from_time) * (i - from_seqno) / + (to_seqno - from_seqno); + pairs_.emplace_back(i, t); + } + + return /*success*/ true; +} + bool SeqnoToTimeMapping::Resize(uint64_t min_time_duration, uint64_t max_time_duration) { uint64_t new_max_capacity = CalculateMaxCapacity(min_time_duration, max_time_duration); if (new_max_capacity == max_capacity_) { return false; - } else if (new_max_capacity < seqno_time_mapping_.size()) { - uint64_t delta = seqno_time_mapping_.size() - new_max_capacity; - seqno_time_mapping_.erase(seqno_time_mapping_.begin(), - seqno_time_mapping_.begin() + delta); + } else if (new_max_capacity < pairs_.size()) { + uint64_t delta = pairs_.size() - new_max_capacity; + // FIXME: be smarter about how we erase to avoid data falling off the + // front prematurely. + pairs_.erase(pairs_.begin(), pairs_.begin() + delta); } max_capacity_ = new_max_capacity; return true; @@ -258,16 +304,16 @@ Status SeqnoToTimeMapping::Sort() { if (is_sorted_) { return Status::OK(); } - if (seqno_time_mapping_.empty()) { + if (pairs_.empty()) { is_sorted_ = true; return Status::OK(); } - std::deque copy = std::move(seqno_time_mapping_); + std::deque copy = std::move(pairs_); std::sort(copy.begin(), copy.end()); - seqno_time_mapping_.clear(); + pairs_.clear(); // remove seqno = 0, which may have special meaning, like zeroed out data while (copy.front().seqno == 0) { @@ -285,12 +331,12 @@ Status SeqnoToTimeMapping::Sort() { assert(it.seqno > prev.seqno); // If a larger sequence number has an older time which is not useful, skip if (it.time > prev.time) { - seqno_time_mapping_.push_back(prev); + pairs_.push_back(prev); prev = it; } } } - seqno_time_mapping_.emplace_back(prev); + pairs_.emplace_back(prev); is_sorted_ = true; return Status::OK(); @@ -298,7 +344,7 @@ Status SeqnoToTimeMapping::Sort() { std::string SeqnoToTimeMapping::ToHumanString() const { std::string ret; - for (const auto& seq_time : seqno_time_mapping_) { + for (const auto& seq_time : pairs_) { AppendNumberTo(&ret, seq_time.seqno); ret.append("->"); AppendNumberTo(&ret, seq_time.time); @@ -310,13 +356,11 @@ std::string SeqnoToTimeMapping::ToHumanString() const { SeqnoToTimeMapping SeqnoToTimeMapping::Copy( SequenceNumber smallest_seqno) const { SeqnoToTimeMapping ret; - auto it = std::upper_bound(seqno_time_mapping_.begin(), - seqno_time_mapping_.end(), smallest_seqno); - if (it != seqno_time_mapping_.begin()) { + auto it = FindGreaterSeqno(smallest_seqno); + if (it != pairs_.begin()) { it--; } - std::copy(it, seqno_time_mapping_.end(), - std::back_inserter(ret.seqno_time_mapping_)); + std::copy(it, pairs_.end(), std::back_inserter(ret.pairs_)); return ret; } @@ -330,12 +374,4 @@ uint64_t SeqnoToTimeMapping::CalculateMaxCapacity(uint64_t min_time_duration, max_time_duration * kMaxSeqnoTimePairsPerCF / min_time_duration); } -SeqnoToTimeMapping::SeqnoTimePair SeqnoToTimeMapping::SeqnoTimePair::operator-( - const SeqnoTimePair& other) const { - SeqnoTimePair res; - res.seqno = seqno - other.seqno; - res.time = time - other.time; - return res; -} - } // namespace ROCKSDB_NAMESPACE diff --git a/db/seqno_to_time_mapping.h b/db/seqno_to_time_mapping.h index 4ffc9c199..95a4455be 100644 --- a/db/seqno_to_time_mapping.h +++ b/db/seqno_to_time_mapping.h @@ -18,20 +18,32 @@ namespace ROCKSDB_NAMESPACE { -constexpr uint64_t kUnknownSeqnoTime = 0; - -// SeqnoToTimeMapping stores the sequence number to time mapping, so given a -// sequence number it can estimate the oldest possible time for that sequence -// number. For example: -// 10 -> 100 -// 50 -> 300 -// then if a key has seqno 19, the OldestApproximateTime would be 100, for 51 it -// would be 300. -// As it's a sorted list, the new entry is inserted from the back. The old data -// will be popped from the front if they're no longer used. +constexpr uint64_t kUnknownTimeBeforeAll = 0; +constexpr SequenceNumber kUnknownSeqnoBeforeAll = 0; + +// SeqnoToTimeMapping stores a sampled mapping from sequence numbers to +// unix times (seconds since epoch). This information provides rough bounds +// between sequence numbers and their write times, but is primarily designed +// for getting a best lower bound on the sequence number of data written no +// later than a specified time. // -// Note: the data struct is not thread safe, both read and write need to be -// synchronized by caller. +// For ease of sampling, it is assumed that the recorded time in each pair +// comes at or after the sequence number and before the next sequence number, +// so this example: +// +// Seqno: 10, 11, ... 20, 21, ... 30, 31, ... +// Time: ... 500 ... 600 ... 700 ... +// +// would be represented as +// 10 -> 500 +// 20 -> 600 +// 30 -> 700 +// +// In typical operation, the list is sorted, both among seqnos and among times, +// with a bounded number of entries, but some public working states violate +// these constraints. +// +// NOT thread safe - requires external synchronization. class SeqnoToTimeMapping { public: // Maximum number of entries can be encoded into SST. The data is delta encode @@ -63,28 +75,33 @@ class SeqnoToTimeMapping { // Decode the value from input Slice and remove it from the input Status Decode(Slice& input); - // subtraction of 2 SeqnoTimePair - SeqnoTimePair operator-(const SeqnoTimePair& other) const; - - // Add 2 values together - void Add(const SeqnoTimePair& obj) { - seqno += obj.seqno; - time += obj.time; + // For delta encoding + SeqnoTimePair ComputeDelta(const SeqnoTimePair& base) const { + return {seqno - base.seqno, time - base.time}; } - // Compare SeqnoTimePair with a sequence number, used for binary search a - // sequence number in a list of SeqnoTimePair - bool operator<(const SequenceNumber& other) const { return seqno < other; } + // For delta decoding + void ApplyDelta(const SeqnoTimePair& delta_or_base) { + seqno += delta_or_base.seqno; + time += delta_or_base.time; + } - // Compare 2 SeqnoTimePair + // Ordering used for Sort() bool operator<(const SeqnoTimePair& other) const { return std::tie(seqno, time) < std::tie(other.seqno, other.time); } - // Check if 2 SeqnoTimePair is the same bool operator==(const SeqnoTimePair& other) const { return std::tie(seqno, time) == std::tie(other.seqno, other.time); } + + static bool SeqnoLess(const SeqnoTimePair& a, const SeqnoTimePair& b) { + return a.seqno < b.seqno; + } + + static bool TimeLess(const SeqnoTimePair& a, const SeqnoTimePair& b) { + return a.time < b.time; + } }; // constractor of SeqnoToTimeMapping @@ -99,20 +116,40 @@ class SeqnoToTimeMapping { uint64_t max_capacity = 0) : max_time_duration_(max_time_duration), max_capacity_(max_capacity) {} + // Both seqno range and time range are inclusive. ... TODO + // + bool PrePopulate(SequenceNumber from_seqno, SequenceNumber to_seqno, + uint64_t from_time, uint64_t to_time); + // Append a new entry to the list. The new entry should be newer than the // existing ones. It maintains the internal sorted status. bool Append(SequenceNumber seqno, uint64_t time); - // Given a sequence number, estimate it's oldest time - uint64_t GetOldestApproximateTime(SequenceNumber seqno) const; - - // Truncate the old entries based on the current time and max_time_duration_ + // Given a sequence number, return the best (largest / newest) known time + // that is no later than the write time of that given sequence number. + // If no such specific time is known, returns kUnknownTimeBeforeAll. + // Using the example in the class comment above, + // GetProximalTimeBeforeSeqno(10) -> kUnknownTimeBeforeAll + // GetProximalTimeBeforeSeqno(11) -> 500 + // GetProximalTimeBeforeSeqno(20) -> 500 + // GetProximalTimeBeforeSeqno(21) -> 600 + uint64_t GetProximalTimeBeforeSeqno(SequenceNumber seqno) const; + + // Remove any entries not needed for GetProximalSeqnoBeforeTime queries of + // times older than `now - max_time_duration_` void TruncateOldEntries(uint64_t now); - // Given a time, return it's oldest possible sequence number - SequenceNumber GetOldestSequenceNum(uint64_t time); - - // Encode to a binary string + // Given a time, return the best (largest) sequence number whose write time + // is no later than that given time. If no such specific sequence number is + // known, returns kUnknownSeqnoBeforeAll. Using the example in the class + // comment above, + // GetProximalSeqnoBeforeTime(499) -> kUnknownSeqnoBeforeAll + // GetProximalSeqnoBeforeTime(500) -> 10 + // GetProximalSeqnoBeforeTime(599) -> 10 + // GetProximalSeqnoBeforeTime(600) -> 20 + SequenceNumber GetProximalSeqnoBeforeTime(uint64_t time); + + // Encode to a binary string. start and end seqno are both inclusive. void Encode(std::string& des, SequenceNumber start, SequenceNumber end, uint64_t now, uint64_t output_size = kMaxSeqnoTimePairsPerSST) const; @@ -122,10 +159,10 @@ class SeqnoToTimeMapping { void Add(SequenceNumber seqno, uint64_t time); // Decode and add the entries to the current obj. The list will be unsorted - Status Add(const std::string& seqno_time_mapping_str); + Status Add(const std::string& pairs_str); // Return the number of entries - size_t Size() const { return seqno_time_mapping_.size(); } + size_t Size() const { return pairs_.size(); } // Reduce the size of internal list bool Resize(uint64_t min_time_duration, uint64_t max_time_duration); @@ -145,10 +182,10 @@ class SeqnoToTimeMapping { SeqnoToTimeMapping Copy(SequenceNumber smallest_seqno) const; // If the internal list is empty - bool Empty() const { return seqno_time_mapping_.empty(); } + bool Empty() const { return pairs_.empty(); } // clear all entries - void Clear() { seqno_time_mapping_.clear(); } + void Clear() { pairs_.clear(); } // return the string for user message // Note: Not efficient, okay for print @@ -156,7 +193,7 @@ class SeqnoToTimeMapping { #ifndef NDEBUG const std::deque& TEST_GetInternalMapping() const { - return seqno_time_mapping_; + return pairs_; } #endif @@ -167,7 +204,7 @@ class SeqnoToTimeMapping { uint64_t max_time_duration_; uint64_t max_capacity_; - std::deque seqno_time_mapping_; + std::deque pairs_; bool is_sorted_ = true; @@ -176,14 +213,14 @@ class SeqnoToTimeMapping { SeqnoTimePair& Last() { assert(!Empty()); - return seqno_time_mapping_.back(); + return pairs_.back(); } -}; -// for searching the sequence number from SeqnoToTimeMapping -inline bool operator<(const SequenceNumber& seqno, - const SeqnoToTimeMapping::SeqnoTimePair& other) { - return seqno < other.seqno; -} + using pair_const_iterator = + std::deque::const_iterator; + pair_const_iterator FindGreaterTime(uint64_t time) const; + pair_const_iterator FindGreaterSeqno(SequenceNumber seqno) const; + pair_const_iterator FindGreaterEqSeqno(SequenceNumber seqno) const; +}; } // namespace ROCKSDB_NAMESPACE diff --git a/db/table_cache.cc b/db/table_cache.cc index 00224fd81..2c0092e7d 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -85,7 +85,7 @@ TableCache::TableCache(const ImmutableOptions& ioptions, } } -TableCache::~TableCache() {} +TableCache::~TableCache() = default; Status TableCache::GetTableReader( const ReadOptions& ro, const FileOptions& file_options, @@ -129,6 +129,10 @@ Status TableCache::GetTableReader( if (!sequential_mode && ioptions_.advise_random_on_open) { file->Hint(FSRandomAccessFile::kRandom); } + if (ioptions_.default_temperature != Temperature::kUnknown && + file_temperature == Temperature::kUnknown) { + file_temperature = ioptions_.default_temperature; + } StopWatch sw(ioptions_.clock, ioptions_.stats, TABLE_OPEN_IO_MICROS); std::unique_ptr file_reader( new RandomAccessFileReader(std::move(file), fname, ioptions_.clock, @@ -220,7 +224,7 @@ InternalIterator* TableCache::NewIterator( size_t max_file_size_for_l0_meta_pin, const InternalKey* smallest_compaction_key, const InternalKey* largest_compaction_key, bool allow_unprepared_value, - uint8_t block_protection_bytes_per_key, + uint8_t block_protection_bytes_per_key, const SequenceNumber* read_seqno, TruncatedRangeDelIterator** range_del_iter) { PERF_TIMER_GUARD(new_table_iterator_nanos); @@ -269,7 +273,9 @@ InternalIterator* TableCache::NewIterator( if (s.ok() && !options.ignore_range_deletions) { if (range_del_iter != nullptr) { auto new_range_del_iter = - table_reader->NewRangeTombstoneIterator(options); + read_seqno ? table_reader->NewRangeTombstoneIterator( + *read_seqno, options.timestamp) + : table_reader->NewRangeTombstoneIterator(options); if (new_range_del_iter == nullptr || new_range_del_iter->empty()) { delete new_range_del_iter; *range_del_iter = nullptr; @@ -345,23 +351,25 @@ Status TableCache::GetRangeTombstoneIterator( return s; } -void TableCache::CreateRowCacheKeyPrefix(const ReadOptions& options, - const FileDescriptor& fd, - const Slice& internal_key, - GetContext* get_context, - IterKey& row_cache_key) { +uint64_t TableCache::CreateRowCacheKeyPrefix(const ReadOptions& options, + const FileDescriptor& fd, + const Slice& internal_key, + GetContext* get_context, + IterKey& row_cache_key) { uint64_t fd_number = fd.GetNumber(); // We use the user key as cache key instead of the internal key, // otherwise the whole cache would be invalidated every time the // sequence key increases. However, to support caching snapshot - // reads, we append the sequence number (incremented by 1 to - // distinguish from 0) only in this case. + // reads, we append a sequence number (incremented by 1 to + // distinguish from 0) other than internal_key seq no + // to determine row cache entry visibility. // If the snapshot is larger than the largest seqno in the file, // all data should be exposed to the snapshot, so we treat it // the same as there is no snapshot. The exception is that if // a seq-checking callback is registered, some internal keys // may still be filtered out. - uint64_t seq_no = 0; + uint64_t cache_entry_seq_no = 0; + // Maybe we can include the whole file ifsnapshot == fd.largest_seqno. if (options.snapshot != nullptr && (get_context->has_callback() || @@ -370,18 +378,24 @@ void TableCache::CreateRowCacheKeyPrefix(const ReadOptions& options, // We should consider to use options.snapshot->GetSequenceNumber() // instead of GetInternalKeySeqno(k), which will make the code // easier to understand. - seq_no = 1 + GetInternalKeySeqno(internal_key); + cache_entry_seq_no = 1 + GetInternalKeySeqno(internal_key); } // Compute row cache key. row_cache_key.TrimAppend(row_cache_key.Size(), row_cache_id_.data(), row_cache_id_.size()); AppendVarint64(&row_cache_key, fd_number); - AppendVarint64(&row_cache_key, seq_no); + AppendVarint64(&row_cache_key, cache_entry_seq_no); + + // Provide a sequence number for callback checking on cache hit. + // As cache_entry_seq_no starts at 1, decrease it's value by 1 to get + // a sequence number align with get context's logic. + return cache_entry_seq_no == 0 ? 0 : cache_entry_seq_no - 1; } bool TableCache::GetFromRowCache(const Slice& user_key, IterKey& row_cache_key, - size_t prefix_size, GetContext* get_context) { + size_t prefix_size, GetContext* get_context, + SequenceNumber seq_no) { bool found = false; row_cache_key.TrimAppend(prefix_size, user_key.data(), user_key.size()); @@ -398,8 +412,10 @@ bool TableCache::GetFromRowCache(const Slice& user_key, IterKey& row_cache_key, // get_context.pinnable_slice_. Cache entry is released when // get_context.pinnable_slice_ is reset. row_cache.RegisterReleaseAsCleanup(row_handle, value_pinner); + // If row cache hit, knowing cache key is the same to row_cache_key, + // can use row_cache_key's seq no to construct InternalKey. replayGetContextLog(*row_cache.Value(row_handle), user_key, get_context, - &value_pinner); + &value_pinner, seq_no); RecordTick(ioptions_.stats, ROW_CACHE_HIT); found = true; } else { @@ -422,13 +438,14 @@ Status TableCache::Get( IterKey row_cache_key; std::string row_cache_entry_buffer; - // Check row cache if enabled. Since row cache does not currently store - // sequence numbers, we cannot use it if we need to fetch the sequence. + // Check row cache if enabled. + // Reuse row_cache_key sequence number when row cache hits. if (ioptions_.row_cache && !get_context->NeedToReadSequence()) { auto user_key = ExtractUserKey(k); - CreateRowCacheKeyPrefix(options, fd, k, get_context, row_cache_key); + uint64_t cache_entry_seq_no = + CreateRowCacheKeyPrefix(options, fd, k, get_context, row_cache_key); done = GetFromRowCache(user_key, row_cache_key, row_cache_key.Size(), - get_context); + get_context, cache_entry_seq_no); if (!done) { row_cache_entry = &row_cache_entry_buffer; } @@ -484,9 +501,12 @@ Status TableCache::Get( RowCacheInterface row_cache{ioptions_.row_cache.get()}; size_t charge = row_cache_entry->capacity() + sizeof(std::string); auto row_ptr = new std::string(std::move(*row_cache_entry)); - // If row cache is full, it's OK to continue. - row_cache.Insert(row_cache_key.GetUserKey(), row_ptr, charge) - .PermitUncheckedError(); + Status rcs = row_cache.Insert(row_cache_key.GetUserKey(), row_ptr, charge); + if (!rcs.ok()) { + // If row cache is full, it's OK to continue, but we keep ownership of + // row_ptr. + delete row_ptr; + } } if (handle != nullptr) { @@ -709,4 +729,4 @@ uint64_t TableCache::ApproximateSize( return result; } -} // namespace ROCKSDB_NAMESPACE +} // namespace ROCKSDB_NAMESPACE \ No newline at end of file diff --git a/db/table_cache.h b/db/table_cache.h index 39e41cc6c..ae3fc93c3 100644 --- a/db/table_cache.h +++ b/db/table_cache.h @@ -86,6 +86,8 @@ class TableCache { // not cached), depending on the CF options // @param skip_filters Disables loading/accessing the filter block // @param level The level this table is at, -1 for "not set / don't know" + // @param range_del_read_seqno If non-nullptr, will be used to create + // *range_del_iter. InternalIterator* NewIterator( const ReadOptions& options, const FileOptions& toptions, const InternalKeyComparator& internal_comparator, @@ -97,6 +99,7 @@ class TableCache { const InternalKey* smallest_compaction_key, const InternalKey* largest_compaction_key, bool allow_unprepared_value, uint8_t protection_bytes_per_key, + const SequenceNumber* range_del_read_seqno = nullptr, TruncatedRangeDelIterator** range_del_iter = nullptr); // If a seek to internal key "k" in specified file finds an entry, @@ -259,15 +262,18 @@ class TableCache { // Create a key prefix for looking up the row cache. The prefix is of the // format row_cache_id + fd_number + seq_no. Later, the user key can be // appended to form the full key - void CreateRowCacheKeyPrefix(const ReadOptions& options, - const FileDescriptor& fd, - const Slice& internal_key, - GetContext* get_context, IterKey& row_cache_key); + // Return the sequence number that determines the visibility of row_cache_key + uint64_t CreateRowCacheKeyPrefix(const ReadOptions& options, + const FileDescriptor& fd, + const Slice& internal_key, + GetContext* get_context, + IterKey& row_cache_key); // Helper function to lookup the row cache for a key. It appends the // user key to row_cache_key at offset prefix_size bool GetFromRowCache(const Slice& user_key, IterKey& row_cache_key, - size_t prefix_size, GetContext* get_context); + size_t prefix_size, GetContext* get_context, + SequenceNumber seq_no = kMaxSequenceNumber); const ImmutableOptions& ioptions_; const FileOptions& file_options_; @@ -280,4 +286,4 @@ class TableCache { std::string db_session_id_; }; -} // namespace ROCKSDB_NAMESPACE +} // namespace ROCKSDB_NAMESPACE \ No newline at end of file diff --git a/db/table_properties_collector.h b/db/table_properties_collector.h index 968115c3d..9bcec93dd 100644 --- a/db/table_properties_collector.h +++ b/db/table_properties_collector.h @@ -98,8 +98,13 @@ class UserKeyTablePropertiesCollectorFactory TablePropertiesCollectorFactory::Context context; context.column_family_id = column_family_id; context.level_at_creation = level_at_creation; - return new UserKeyTablePropertiesCollector( - user_collector_factory_->CreateTablePropertiesCollector(context)); + TablePropertiesCollector* collector = + user_collector_factory_->CreateTablePropertiesCollector(context); + if (collector) { + return new UserKeyTablePropertiesCollector(collector); + } else { + return nullptr; + } } virtual const char* Name() const override { diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc index 437b7e309..e10f02e67 100644 --- a/db/table_properties_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -180,7 +180,6 @@ class RegularKeysStartWithAInternal : public IntTblPropCollector { uint64_t /* block_compressed_bytes_fast */, uint64_t /* block_compressed_bytes_slow */) override { // Nothing to do. - return; } UserCollectedProperties GetReadableProperties() const override { @@ -227,7 +226,7 @@ class FlushBlockEveryThreePolicy : public FlushBlockPolicy { class FlushBlockEveryThreePolicyFactory : public FlushBlockPolicyFactory { public: - explicit FlushBlockEveryThreePolicyFactory() {} + explicit FlushBlockEveryThreePolicyFactory() = default; const char* Name() const override { return "FlushBlockEveryThreePolicyFactory"; diff --git a/db/version_builder_test.cc b/db/version_builder_test.cc index ac80be7ca..2ca10c449 100644 --- a/db/version_builder_test.cc +++ b/db/version_builder_test.cc @@ -37,7 +37,9 @@ class VersionBuilderTest : public testing::Test { ioptions_(options_), mutable_cf_options_(options_), vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, - nullptr, false), + nullptr, false, EpochNumberRequirement::kMustPresent, + ioptions_.clock, options_.bottommost_file_compaction_delay, + OffpeakTimeOption(options_.daily_offpeak_time_utc)), file_num_(1) { mutable_cf_options_.RefreshDerivedOptions(ioptions_); size_being_compacted_.resize(options_.num_levels); @@ -199,8 +201,10 @@ TEST_F(VersionBuilderTest, ApplyAndSaveTo) { VersionBuilder version_builder(env_options, &ioptions_, table_cache, &vstorage_, version_set); - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, nullptr, false); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.SaveTo(&new_vstorage)); @@ -249,8 +253,10 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) { VersionBuilder version_builder(env_options, &ioptions_, table_cache, &vstorage_, version_set); - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, nullptr, false); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.SaveTo(&new_vstorage)); @@ -303,8 +309,10 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) { VersionBuilder version_builder(env_options, &ioptions_, table_cache, &vstorage_, version_set); - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, nullptr, false); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.SaveTo(&new_vstorage)); @@ -359,8 +367,10 @@ TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) { VersionBuilder version_builder(env_options, &ioptions_, table_cache, &vstorage_, version_set); - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, nullptr, false); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.SaveTo(&new_vstorage)); @@ -381,8 +391,10 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) { VersionBuilder version_builder(env_options, &ioptions_, table_cache, &vstorage_, version_set); - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, nullptr, false); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); VersionEdit version_edit; version_edit.AddFile( @@ -548,9 +560,10 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) { ASSERT_OK(builder.Apply(&addition)); constexpr bool force_consistency_checks = false; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -692,9 +705,10 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) { ASSERT_OK(builder.Apply(&deletion)); constexpr bool force_consistency_checks = false; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -736,9 +750,10 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAddition) { ASSERT_OK(builder.Apply(&edit)); constexpr bool force_consistency_checks = false; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -875,9 +890,10 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) { ASSERT_OK(builder.Apply(&edit)); constexpr bool force_consistency_checks = false; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -948,9 +964,10 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) { ASSERT_OK(builder.Apply(&garbage)); constexpr bool force_consistency_checks = false; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -1128,9 +1145,10 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) { ASSERT_OK(builder.Apply(&edit)); constexpr bool force_consistency_checks = false; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -1176,9 +1194,10 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) { ASSERT_OK(second_builder.Apply(&second_edit)); - VersionStorageInfo newer_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &new_vstorage, - force_consistency_checks); + VersionStorageInfo newer_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &new_vstorage, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(second_builder.SaveTo(&newer_vstorage)); @@ -1262,9 +1281,10 @@ TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) { ASSERT_OK(builder.Apply(&edit)); constexpr bool force_consistency_checks = true; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -1365,9 +1385,10 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) { // Save to a new version in order to trigger consistency checks. constexpr bool force_consistency_checks = true; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -1404,9 +1425,10 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesInconsistentLinks) { // Save to a new version in order to trigger consistency checks. constexpr bool force_consistency_checks = true; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); const Status s = builder.SaveTo(&new_vstorage); ASSERT_TRUE(s.IsCorruption()); @@ -1445,9 +1467,10 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbage) { // Save to a new version in order to trigger consistency checks. constexpr bool force_consistency_checks = true; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); const Status s = builder.SaveTo(&new_vstorage); ASSERT_TRUE(s.IsCorruption()); @@ -1494,9 +1517,10 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbageLinkedSsts) { // Save to a new version in order to trigger consistency checks. constexpr bool force_consistency_checks = true; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); const Status s = builder.SaveTo(&new_vstorage); ASSERT_TRUE(s.IsCorruption()); @@ -1657,9 +1681,10 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) { ASSERT_OK(builder.Apply(&edit2)); constexpr bool force_consistency_checks = true; - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, &vstorage_, - force_consistency_checks); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + force_consistency_checks, EpochNumberRequirement::kMightMissing, nullptr, + 0, OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(builder.SaveTo(&new_vstorage)); @@ -1708,9 +1733,11 @@ TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) { VersionBuilder version_builder(env_options, &ioptions_, table_cache, &vstorage_, version_set); - VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, nullptr, - true /* force_consistency_checks */); + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, + true /* force_consistency_checks */, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(version_builder.Apply(&version_edit)); ASSERT_OK(version_builder.SaveTo(&new_vstorage)); @@ -1718,9 +1745,11 @@ TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) { VersionBuilder version_builder2(env_options, &ioptions_, table_cache, &new_vstorage, version_set); - VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels, - kCompactionStyleLevel, nullptr, - true /* force_consistency_checks */); + VersionStorageInfo new_vstorage2( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, + true /* force_consistency_checks */, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_NOK(version_builder2.Apply(&version_edit)); UnrefFilesInVersion(&new_vstorage); @@ -1758,7 +1787,9 @@ TEST_F(VersionBuilderTest, CheckConsistencyForL0FilesSortedByEpochNumber) { nullptr /* file_metadata_cache_res_mgr */); VersionStorageInfo new_vstorage_1( &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, - nullptr /* src_vstorage */, true /* force_consistency_checks */); + nullptr /* src_vstorage */, true /* force_consistency_checks */, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(version_builder_1.Apply(&version_edit_1)); s = version_builder_1.SaveTo(&new_vstorage_1); @@ -1795,7 +1826,9 @@ TEST_F(VersionBuilderTest, CheckConsistencyForL0FilesSortedByEpochNumber) { nullptr /* file_metadata_cache_res_mgr */); VersionStorageInfo new_vstorage_2( &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, - nullptr /* src_vstorage */, true /* force_consistency_checks */); + nullptr /* src_vstorage */, true /* force_consistency_checks */, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc)); ASSERT_OK(version_builder_2.Apply(&version_edit_2)); s = version_builder_2.SaveTo(&new_vstorage_2); diff --git a/db/version_edit.cc b/db/version_edit.cc index f5783eacd..482aa65a7 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -100,6 +100,7 @@ bool VersionEdit::EncodeTo(std::string* dst, PutLengthPrefixedSlice(dst, db_id_); } if (has_comparator_) { + assert(has_persist_user_defined_timestamps_); PutVarint32(dst, kComparator); PutLengthPrefixedSlice(dst, comparator_); } @@ -308,6 +309,15 @@ bool VersionEdit::EncodeTo(std::string* dst, PutVarint32(dst, kFullHistoryTsLow); PutLengthPrefixedSlice(dst, full_history_ts_low_); } + + if (HasPersistUserDefinedTimestamps()) { + // persist_user_defined_timestamps flag should be logged in the same + // VersionEdit as the user comparator name. + assert(has_comparator_); + PutVarint32(dst, kPersistUserDefinedTimestamps); + char p = static_cast(persist_user_defined_timestamps_); + PutLengthPrefixedSlice(dst, Slice(&p, 1)); + } return true; } @@ -474,7 +484,6 @@ void VersionEdit::EncodeFileBoundaries(std::string* dst, StripTimestampFromInternalKey(&largest_buf, meta.largest.Encode(), ts_sz); PutLengthPrefixedSlice(dst, smallest_buf); PutLengthPrefixedSlice(dst, largest_buf); - return; }; Status VersionEdit::DecodeFrom(const Slice& src) { @@ -777,6 +786,17 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } break; + case kPersistUserDefinedTimestamps: + if (!GetLengthPrefixedSlice(&input, &str)) { + msg = "persist_user_defined_timestamps"; + } else if (str.size() != 1) { + msg = "persist_user_defined_timestamps field wrong size"; + } else { + persist_user_defined_timestamps_ = (str[0] == 1); + has_persist_user_defined_timestamps_ = true; + } + break; + default: if (tag & kTagSafeIgnoreMask) { // Tag from future which can be safely ignored. @@ -819,6 +839,10 @@ std::string VersionEdit::DebugString(bool hex_key) const { r.append("\n Comparator: "); r.append(comparator_); } + if (has_persist_user_defined_timestamps_) { + r.append("\n PersistUserDefinedTimestamps: "); + r.append(persist_user_defined_timestamps_ ? "true" : "false"); + } if (has_log_number_) { r.append("\n LogNumber: "); AppendNumberTo(&r, log_number_); diff --git a/db/version_edit.h b/db/version_edit.h index cedccb3a2..8e14e76da 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -71,6 +71,7 @@ enum Tag : uint32_t { kFullHistoryTsLow, kWalAddition2, kWalDeletion2, + kPersistUserDefinedTimestamps, }; enum NewFileCustomTag : uint32_t { @@ -192,7 +193,8 @@ struct FileMetaData { uint64_t compensated_file_size = 0; // These values can mutate, but they can only be read or written from // single-threaded LogAndApply thread - uint64_t num_entries = 0; // the number of entries. + uint64_t num_entries = + 0; // The number of entries, including deletions and range deletions. // The number of deletion entries, including range deletions. uint64_t num_deletions = 0; uint64_t raw_key_size = 0; // total uncompressed key size. @@ -217,10 +219,16 @@ struct FileMetaData { // refers to. 0 is an invalid value; BlobDB numbers the files starting from 1. uint64_t oldest_blob_file_number = kInvalidBlobFileNumber; - // The file could be the compaction output from other SST files, which could - // in turn be outputs for compact older SST files. We track the memtable - // flush timestamp for the oldest SST file that eventually contribute data - // to this file. 0 means the information is not available. + // For flush output file, oldest ancestor time is the oldest key time in the + // file. If the oldest key time is not available, flush time is used. + // + // For compaction output file, oldest ancestor time is the oldest + // among all the oldest key time of its input files, since the file could be + // the compaction output from other SST files, which could in turn be outputs + // for compact older SST files. If that's not available, creation time of this + // compaction output file is used. + // + // 0 means the information is not available. uint64_t oldest_ancester_time = kUnknownOldestAncesterTime; // Unix time when the SST file is created. @@ -397,6 +405,17 @@ class VersionEdit { bool HasComparatorName() const { return has_comparator_; } const std::string& GetComparatorName() const { return comparator_; } + void SetPersistUserDefinedTimestamps(bool persist_user_defined_timestamps) { + has_persist_user_defined_timestamps_ = true; + persist_user_defined_timestamps_ = persist_user_defined_timestamps; + } + bool HasPersistUserDefinedTimestamps() const { + return has_persist_user_defined_timestamps_; + } + bool GetPersistUserDefinedTimestamps() const { + return persist_user_defined_timestamps_; + } + void SetLogNumber(uint64_t num) { has_log_number_ = true; log_number_ = num; @@ -473,6 +492,7 @@ class VersionEdit { file_checksum_func_name, unique_id, compensated_range_deletion_size, tail_size, user_defined_timestamps_persisted)); + files_to_quarantine_.push_back(file); if (!HasLastSequence() || largest_seqno > GetLastSequence()) { SetLastSequence(largest_seqno); } @@ -481,6 +501,7 @@ class VersionEdit { void AddFile(int level, const FileMetaData& f) { assert(f.fd.smallest_seqno <= f.fd.largest_seqno); new_files_.emplace_back(level, f); + files_to_quarantine_.push_back(f.fd.GetNumber()); if (!HasLastSequence() || f.fd.largest_seqno > GetLastSequence()) { SetLastSequence(f.fd.largest_seqno); } @@ -517,10 +538,13 @@ class VersionEdit { blob_file_additions_.emplace_back( blob_file_number, total_blob_count, total_blob_bytes, std::move(checksum_method), std::move(checksum_value)); + files_to_quarantine_.push_back(blob_file_number); } void AddBlobFile(BlobFileAddition blob_file_addition) { blob_file_additions_.emplace_back(std::move(blob_file_addition)); + files_to_quarantine_.push_back( + blob_file_additions_.back().GetBlobFileNumber()); } // Retrieve all the blob files added. @@ -532,6 +556,11 @@ class VersionEdit { void SetBlobFileAdditions(BlobFileAdditions blob_file_additions) { assert(blob_file_additions_.empty()); blob_file_additions_ = std::move(blob_file_additions); + std::for_each( + blob_file_additions_.begin(), blob_file_additions_.end(), + [&](const BlobFileAddition& blob_file) { + files_to_quarantine_.push_back(blob_file.GetBlobFileNumber()); + }); } // Add garbage for an existing blob file. Note: intentionally broken English @@ -599,6 +628,8 @@ class VersionEdit { } uint32_t GetColumnFamily() const { return column_family_; } + const std::string& GetColumnFamilyName() const { return column_family_name_; } + // set column family ID by calling SetColumnFamily() void AddColumnFamily(const std::string& name) { assert(!is_column_family_drop_); @@ -629,6 +660,9 @@ class VersionEdit { remaining_entries_ = remaining_entries; } bool IsInAtomicGroup() const { return is_in_atomic_group_; } + void SetRemainingEntries(uint32_t remaining_entries) { + remaining_entries_ = remaining_entries; + } uint32_t GetRemainingEntries() const { return remaining_entries_; } bool HasFullHistoryTsLow() const { return !full_history_ts_low_.empty(); } @@ -655,20 +689,14 @@ class VersionEdit { std::optional ts_sz = std::nullopt) const; Status DecodeFrom(const Slice& src); + const autovector* GetFilesToQuarantineIfCommitFail() const { + return &files_to_quarantine_; + } + std::string DebugString(bool hex_key = false) const; std::string DebugJSON(int edit_num, bool hex_key = false) const; private: - friend class ReactiveVersionSet; - friend class VersionEditHandlerBase; - friend class ListColumnFamiliesHandler; - friend class VersionEditHandler; - friend class VersionEditHandlerPointInTime; - friend class DumpManifestHandler; - friend class VersionSet; - friend class Version; - friend class AtomicGroupReadBuffer; - bool GetLevel(Slice* input, int* level, const char** msg); const char* DecodeNewFile4From(Slice* input); @@ -697,6 +725,7 @@ class VersionEdit { bool has_max_column_family_ = false; bool has_min_log_number_to_keep_ = false; bool has_last_sequence_ = false; + bool has_persist_user_defined_timestamps_ = false; // Compaction cursors for round-robin compaction policy CompactCursors compact_cursors_; @@ -724,6 +753,17 @@ class VersionEdit { uint32_t remaining_entries_ = 0; std::string full_history_ts_low_; + bool persist_user_defined_timestamps_ = true; + + // Newly created table files and blob files are eligible for deletion if they + // are not registered as live files after the background jobs creating them + // have finished. In case committing the VersionEdit containing such changes + // to manifest encountered an error, we want to quarantine these files from + // deletion to avoid prematurely deleting files that ended up getting recorded + // in Manifest as live files. + // Since table files and blob files share the same file number space, we just + // record the file number here. + autovector files_to_quarantine_; }; } // namespace ROCKSDB_NAMESPACE diff --git a/db/version_edit_handler.cc b/db/version_edit_handler.cc index 732723996..90afc0938 100644 --- a/db/version_edit_handler.cc +++ b/db/version_edit_handler.cc @@ -17,6 +17,7 @@ #include "db/version_edit.h" #include "logging/logging.h" #include "monitoring/persistent_stats_history.h" +#include "util/udt_util.h" namespace ROCKSDB_NAMESPACE { @@ -42,7 +43,7 @@ void VersionEditHandlerBase::Iterate(log::Reader& reader, break; } ColumnFamilyData* cfd = nullptr; - if (edit.is_in_atomic_group_) { + if (edit.IsInAtomicGroup()) { if (read_buffer_.IsFull()) { for (auto& e : read_buffer_.replay_buffer()) { s = ApplyVersionEdit(e, &cfd); @@ -100,20 +101,18 @@ void VersionEditHandlerBase::Iterate(log::Reader& reader, Status ListColumnFamiliesHandler::ApplyVersionEdit( VersionEdit& edit, ColumnFamilyData** /*unused*/) { Status s; - if (edit.is_column_family_add_) { - if (column_family_names_.find(edit.column_family_) != - column_family_names_.end()) { + uint32_t cf_id = edit.GetColumnFamily(); + if (edit.IsColumnFamilyAdd()) { + if (column_family_names_.find(cf_id) != column_family_names_.end()) { s = Status::Corruption("Manifest adding the same column family twice"); } else { - column_family_names_.insert( - {edit.column_family_, edit.column_family_name_}); + column_family_names_.insert({cf_id, edit.GetColumnFamilyName()}); } - } else if (edit.is_column_family_drop_) { - if (column_family_names_.find(edit.column_family_) == - column_family_names_.end()) { + } else if (edit.IsColumnFamilyDrop()) { + if (column_family_names_.find(cf_id) == column_family_names_.end()) { s = Status::Corruption("Manifest - dropping non-existing column family"); } else { - column_family_names_.erase(edit.column_family_); + column_family_names_.erase(cf_id); } } return s; @@ -200,9 +199,9 @@ Status VersionEditHandler::Initialize() { Status VersionEditHandler::ApplyVersionEdit(VersionEdit& edit, ColumnFamilyData** cfd) { Status s; - if (edit.is_column_family_add_) { + if (edit.IsColumnFamilyAdd()) { s = OnColumnFamilyAdd(edit, cfd); - } else if (edit.is_column_family_drop_) { + } else if (edit.IsColumnFamilyDrop()) { s = OnColumnFamilyDrop(edit, cfd); } else if (edit.IsWalAddition()) { s = OnWalAddition(edit); @@ -226,22 +225,22 @@ Status VersionEditHandler::OnColumnFamilyAdd(VersionEdit& edit, assert(cfd != nullptr); *cfd = nullptr; + const std::string& cf_name = edit.GetColumnFamilyName(); Status s; if (cf_in_builders || cf_in_not_found) { s = Status::Corruption("MANIFEST adding the same column family twice: " + - edit.column_family_name_); + cf_name); } if (s.ok()) { - auto cf_options = name_to_options_.find(edit.column_family_name_); + auto cf_options = name_to_options_.find(cf_name); // implicitly add persistent_stats column family without requiring user // to specify ColumnFamilyData* tmp_cfd = nullptr; bool is_persistent_stats_column_family = - edit.column_family_name_.compare(kPersistentStatsColumnFamilyName) == 0; + cf_name.compare(kPersistentStatsColumnFamilyName) == 0; if (cf_options == name_to_options_.end() && !is_persistent_stats_column_family) { - column_families_not_found_.emplace(edit.column_family_, - edit.column_family_name_); + column_families_not_found_.emplace(edit.GetColumnFamily(), cf_name); } else { if (is_persistent_stats_column_family) { ColumnFamilyOptions cfo; @@ -269,7 +268,7 @@ Status VersionEditHandler::OnColumnFamilyDrop(VersionEdit& edit, if (cf_in_builders) { tmp_cfd = DestroyCfAndCleanup(edit); } else if (cf_in_not_found) { - column_families_not_found_.erase(edit.column_family_); + column_families_not_found_.erase(edit.GetColumnFamily()); } else { s = Status::Corruption("MANIFEST - dropping non-existing column family"); } @@ -304,10 +303,10 @@ Status VersionEditHandler::OnNonCfOperation(VersionEdit& edit, } ColumnFamilyData* tmp_cfd = nullptr; if (s.ok()) { - auto builder_iter = builders_.find(edit.column_family_); + auto builder_iter = builders_.find(edit.GetColumnFamily()); assert(builder_iter != builders_.end()); tmp_cfd = version_set_->GetColumnFamilySet()->GetColumnFamily( - edit.column_family_); + edit.GetColumnFamily()); assert(tmp_cfd != nullptr); // It's important to handle file boundaries before `MaybeCreateVersion` // because `VersionEditHandlerPointInTime::MaybeCreateVersion` does @@ -361,11 +360,12 @@ void VersionEditHandler::CheckColumnFamilyId(const VersionEdit& edit, // record. Once we encounter column family drop record, // we will delete the column family from // column_families_not_found. - bool in_not_found = column_families_not_found_.find(edit.column_family_) != + uint32_t cf_id = edit.GetColumnFamily(); + bool in_not_found = column_families_not_found_.find(cf_id) != column_families_not_found_.end(); // in builders means that user supplied that column family // option AND that we encountered column family add record - bool in_builders = builders_.find(edit.column_family_) != builders_.end(); + bool in_builders = builders_.find(cf_id) != builders_.end(); // They cannot both be true assert(!(in_not_found && in_builders)); *cf_in_not_found = in_not_found; @@ -377,17 +377,17 @@ void VersionEditHandler::CheckIterationResult(const log::Reader& reader, assert(s != nullptr); if (!s->ok()) { // Do nothing here. - } else if (!version_edit_params_.has_log_number_ || - !version_edit_params_.has_next_file_number_ || - !version_edit_params_.has_last_sequence_) { + } else if (!version_edit_params_.HasLogNumber() || + !version_edit_params_.HasNextFile() || + !version_edit_params_.HasLastSequence()) { std::string msg("no "); - if (!version_edit_params_.has_log_number_) { + if (!version_edit_params_.HasLogNumber()) { msg.append("log_file_number, "); } - if (!version_edit_params_.has_next_file_number_) { + if (!version_edit_params_.HasNextFile()) { msg.append("next_file_number, "); } - if (!version_edit_params_.has_last_sequence_) { + if (!version_edit_params_.HasLastSequence()) { msg.append("last_sequence, "); } msg = msg.substr(0, msg.size() - 2); @@ -408,11 +408,11 @@ void VersionEditHandler::CheckIterationResult(const log::Reader& reader, } if (s->ok()) { version_set_->GetColumnFamilySet()->UpdateMaxColumnFamily( - version_edit_params_.max_column_family_); + version_edit_params_.GetMaxColumnFamily()); version_set_->MarkMinLogNumberToKeep( - version_edit_params_.min_log_number_to_keep_); - version_set_->MarkFileNumberUsed(version_edit_params_.prev_log_number_); - version_set_->MarkFileNumberUsed(version_edit_params_.log_number_); + version_edit_params_.GetMinLogNumberToKeep()); + version_set_->MarkFileNumberUsed(version_edit_params_.GetPrevLogNumber()); + version_set_->MarkFileNumberUsed(version_edit_params_.GetLogNumber()); for (auto* cfd : *(version_set_->GetColumnFamilySet())) { if (cfd->IsDropped()) { continue; @@ -463,9 +463,9 @@ void VersionEditHandler::CheckIterationResult(const log::Reader& reader, if (s->ok()) { version_set_->manifest_file_size_ = reader.GetReadOffset(); assert(version_set_->manifest_file_size_ > 0); - version_set_->next_file_number_.store( - version_edit_params_.next_file_number_ + 1); - SequenceNumber last_seq = version_edit_params_.last_sequence_; + version_set_->next_file_number_.store(version_edit_params_.GetNextFile() + + 1); + SequenceNumber last_seq = version_edit_params_.GetLastSequence(); assert(last_seq != kMaxSequenceNumber); if (last_seq != kMaxSequenceNumber && last_seq > version_set_->last_allocated_sequence_.load()) { @@ -487,46 +487,46 @@ void VersionEditHandler::CheckIterationResult(const log::Reader& reader, // sequence number zeroed through compaction. version_set_->descriptor_last_sequence_ = last_seq; } - version_set_->prev_log_number_ = version_edit_params_.prev_log_number_; + version_set_->prev_log_number_ = version_edit_params_.GetPrevLogNumber(); } } ColumnFamilyData* VersionEditHandler::CreateCfAndInit( const ColumnFamilyOptions& cf_options, const VersionEdit& edit) { + uint32_t cf_id = edit.GetColumnFamily(); ColumnFamilyData* cfd = version_set_->CreateColumnFamily(cf_options, read_options_, &edit); assert(cfd != nullptr); cfd->set_initialized(); - assert(builders_.find(edit.column_family_) == builders_.end()); - builders_.emplace(edit.column_family_, + assert(builders_.find(cf_id) == builders_.end()); + builders_.emplace(cf_id, VersionBuilderUPtr(new BaseReferencedVersionBuilder(cfd))); if (track_missing_files_) { - cf_to_missing_files_.emplace(edit.column_family_, - std::unordered_set()); - cf_to_missing_blob_files_high_.emplace(edit.column_family_, - kInvalidBlobFileNumber); + cf_to_missing_files_.emplace(cf_id, std::unordered_set()); + cf_to_missing_blob_files_high_.emplace(cf_id, kInvalidBlobFileNumber); } return cfd; } ColumnFamilyData* VersionEditHandler::DestroyCfAndCleanup( const VersionEdit& edit) { - auto builder_iter = builders_.find(edit.column_family_); + uint32_t cf_id = edit.GetColumnFamily(); + auto builder_iter = builders_.find(cf_id); assert(builder_iter != builders_.end()); builders_.erase(builder_iter); if (track_missing_files_) { - auto missing_files_iter = cf_to_missing_files_.find(edit.column_family_); + auto missing_files_iter = cf_to_missing_files_.find(cf_id); assert(missing_files_iter != cf_to_missing_files_.end()); cf_to_missing_files_.erase(missing_files_iter); auto missing_blob_files_high_iter = - cf_to_missing_blob_files_high_.find(edit.column_family_); + cf_to_missing_blob_files_high_.find(cf_id); assert(missing_blob_files_high_iter != cf_to_missing_blob_files_high_.end()); cf_to_missing_blob_files_high_.erase(missing_blob_files_high_iter); } ColumnFamilyData* ret = - version_set_->GetColumnFamilySet()->GetColumnFamily(edit.column_family_); + version_set_->GetColumnFamilySet()->GetColumnFamily(cf_id); assert(ret != nullptr); ret->SetDropped(); ret->UnrefAndTryDelete(); @@ -597,30 +597,36 @@ Status VersionEditHandler::LoadTables(ColumnFamilyData* cfd, Status VersionEditHandler::ExtractInfoFromVersionEdit(ColumnFamilyData* cfd, const VersionEdit& edit) { Status s; - if (edit.has_db_id_) { + if (edit.HasDbId()) { version_set_->db_id_ = edit.GetDbId(); - version_edit_params_.SetDBId(edit.db_id_); + version_edit_params_.SetDBId(edit.GetDbId()); } if (cfd != nullptr) { - if (edit.has_log_number_) { - if (cfd->GetLogNumber() > edit.log_number_) { + if (edit.HasLogNumber()) { + if (cfd->GetLogNumber() > edit.GetLogNumber()) { ROCKS_LOG_WARN( version_set_->db_options()->info_log, "MANIFEST corruption detected, but ignored - Log numbers in " "records NOT monotonically increasing"); } else { - cfd->SetLogNumber(edit.log_number_); - version_edit_params_.SetLogNumber(edit.log_number_); + cfd->SetLogNumber(edit.GetLogNumber()); + version_edit_params_.SetLogNumber(edit.GetLogNumber()); } } - if (edit.has_comparator_ && - edit.comparator_ != cfd->user_comparator()->Name()) { - if (!cf_to_cmp_names_) { - s = Status::InvalidArgument( - cfd->user_comparator()->Name(), - "does not match existing comparator " + edit.comparator_); - } else { - cf_to_cmp_names_->emplace(cfd->GetID(), edit.comparator_); + if (edit.HasComparatorName()) { + bool mark_sst_files_has_no_udt = false; + // If `persist_user_defined_timestamps` flag is recorded in manifest, it + // is guaranteed to be in the same VersionEdit as comparator. Otherwise, + // it's not recorded and it should have default value true. + s = ValidateUserDefinedTimestampsOptions( + cfd->user_comparator(), edit.GetComparatorName(), + cfd->ioptions()->persist_user_defined_timestamps, + edit.GetPersistUserDefinedTimestamps(), &mark_sst_files_has_no_udt); + if (!s.ok() && cf_to_cmp_names_) { + cf_to_cmp_names_->emplace(cfd->GetID(), edit.GetComparatorName()); + } + if (mark_sst_files_has_no_udt) { + cfds_to_mark_no_udt_.insert(cfd->GetID()); } } if (edit.HasFullHistoryTsLow()) { @@ -630,29 +636,29 @@ Status VersionEditHandler::ExtractInfoFromVersionEdit(ColumnFamilyData* cfd, } if (s.ok()) { - if (edit.has_prev_log_number_) { - version_edit_params_.SetPrevLogNumber(edit.prev_log_number_); + if (edit.HasPrevLogNumber()) { + version_edit_params_.SetPrevLogNumber(edit.GetPrevLogNumber()); } - if (edit.has_next_file_number_) { - version_edit_params_.SetNextFile(edit.next_file_number_); + if (edit.HasNextFile()) { + version_edit_params_.SetNextFile(edit.GetNextFile()); } - if (edit.has_max_column_family_) { - version_edit_params_.SetMaxColumnFamily(edit.max_column_family_); + if (edit.HasMaxColumnFamily()) { + version_edit_params_.SetMaxColumnFamily(edit.GetMaxColumnFamily()); } - if (edit.has_min_log_number_to_keep_) { - version_edit_params_.min_log_number_to_keep_ = - std::max(version_edit_params_.min_log_number_to_keep_, - edit.min_log_number_to_keep_); + if (edit.HasMinLogNumberToKeep()) { + version_edit_params_.SetMinLogNumberToKeep( + std::max(version_edit_params_.GetMinLogNumberToKeep(), + edit.GetMinLogNumberToKeep())); } - if (edit.has_last_sequence_) { + if (edit.HasLastSequence()) { // `VersionEdit::last_sequence_`s are assumed to be non-decreasing. This // is legacy behavior that cannot change without breaking downgrade // compatibility. - assert(!version_edit_params_.has_last_sequence_ || - version_edit_params_.last_sequence_ <= edit.last_sequence_); - version_edit_params_.SetLastSequence(edit.last_sequence_); + assert(!version_edit_params_.HasLastSequence() || + version_edit_params_.GetLastSequence() <= edit.GetLastSequence()); + version_edit_params_.SetLastSequence(edit.GetLastSequence()); } - if (!version_edit_params_.has_prev_log_number_) { + if (!version_edit_params_.HasPrevLogNumber()) { version_edit_params_.SetPrevLogNumber(0); } } @@ -673,10 +679,17 @@ Status VersionEditHandler::MaybeHandleFileBoundariesForNewFiles( VersionEdit::NewFiles& new_files = edit.GetMutableNewFiles(); assert(!new_files.empty()); + // If true, enabling user-defined timestamp is detected for this column + // family. All its existing SST files need to have the file boundaries handled + // and their `persist_user_defined_timestamps` flag set to false regardless of + // its existing value. + bool mark_existing_ssts_with_no_udt = + cfds_to_mark_no_udt_.find(cfd->GetID()) != cfds_to_mark_no_udt_.end(); bool file_boundaries_need_handling = false; for (auto& new_file : new_files) { FileMetaData& meta = new_file.second; - if (meta.user_defined_timestamps_persisted) { + if (meta.user_defined_timestamps_persisted && + !mark_existing_ssts_with_no_udt) { // `FileMetaData.user_defined_timestamps_persisted` field is the value of // the flag `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` // at the time when the SST file was created. As a result, all added SST @@ -689,6 +702,11 @@ Status VersionEditHandler::MaybeHandleFileBoundariesForNewFiles( break; } file_boundaries_need_handling = true; + assert(!meta.user_defined_timestamps_persisted || + mark_existing_ssts_with_no_udt); + if (mark_existing_ssts_with_no_udt) { + meta.user_defined_timestamps_persisted = false; + } std::string smallest_buf; std::string largest_buf; PadInternalKeyWithMinTimestamp(&smallest_buf, meta.smallest.Encode(), @@ -746,7 +764,7 @@ void VersionEditHandlerPointInTime::CheckIterationResult( ColumnFamilyData* VersionEditHandlerPointInTime::DestroyCfAndCleanup( const VersionEdit& edit) { ColumnFamilyData* cfd = VersionEditHandler::DestroyCfAndCleanup(edit); - auto v_iter = versions_.find(edit.column_family_); + auto v_iter = versions_.find(edit.GetColumnFamily()); if (v_iter != versions_.end()) { delete v_iter->second; versions_.erase(v_iter); @@ -758,7 +776,7 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersion( const VersionEdit& edit, ColumnFamilyData* cfd, bool force_create_version) { assert(cfd != nullptr); if (!force_create_version) { - assert(edit.column_family_ == cfd->GetID()); + assert(edit.GetColumnFamily() == cfd->GetID()); } auto missing_files_iter = cf_to_missing_files_.find(cfd->GetID()); assert(missing_files_iter != cf_to_missing_files_.end()); @@ -841,9 +859,9 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersion( const bool has_missing_files = !missing_files.empty() || has_missing_blob_files; - bool missing_info = !version_edit_params_.has_log_number_ || - !version_edit_params_.has_next_file_number_ || - !version_edit_params_.has_last_sequence_; + bool missing_info = !version_edit_params_.HasLogNumber() || + !version_edit_params_.HasNextFile() || + !version_edit_params_.HasLastSequence(); // Create version before apply edit. The version will represent the state // before applying the version edit. diff --git a/db/version_edit_handler.h b/db/version_edit_handler.h index 54454cf70..af0817e4a 100644 --- a/db/version_edit_handler.h +++ b/db/version_edit_handler.h @@ -133,8 +133,8 @@ class VersionEditHandler : public VersionEditHandlerBase { bool HasMissingFiles() const; void GetDbId(std::string* db_id) const { - if (db_id && version_edit_params_.has_db_id_) { - *db_id = version_edit_params_.db_id_; + if (db_id && version_edit_params_.HasDbId()) { + *db_id = version_edit_params_.GetDbId(); } } @@ -202,6 +202,7 @@ class VersionEditHandler : public VersionEditHandlerBase { bool initialized_; std::unique_ptr> cf_to_cmp_names_; EpochNumberRequirement epoch_number_requirement_; + std::unordered_set cfds_to_mark_no_udt_; private: Status ExtractInfoFromVersionEdit(ColumnFamilyData* cfd, diff --git a/db/version_edit_test.cc b/db/version_edit_test.cc index f3473b476..252352069 100644 --- a/db/version_edit_test.cc +++ b/db/version_edit_test.cc @@ -58,6 +58,7 @@ TEST_F(VersionEditTest, EncodeDecode) { } edit.SetComparatorName("foo"); + edit.SetPersistUserDefinedTimestamps(true); edit.SetLogNumber(kBig + 100); edit.SetNextFile(kBig + 200); edit.SetLastSequence(kBig + 1000); @@ -95,6 +96,7 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) { edit.DeleteFile(4, 700); edit.SetComparatorName("foo"); + edit.SetPersistUserDefinedTimestamps(false); edit.SetLogNumber(kBig + 100); edit.SetNextFile(kBig + 200); edit.SetLastSequence(kBig + 1000); @@ -125,6 +127,7 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) { ASSERT_FALSE(new_files[1].second.user_defined_timestamps_persisted); ASSERT_TRUE(new_files[2].second.user_defined_timestamps_persisted); ASSERT_TRUE(new_files[3].second.user_defined_timestamps_persisted); + ASSERT_FALSE(parsed.GetPersistUserDefinedTimestamps()); } TEST_F(VersionEditTest, EncodeDecodeNewFile4HandleFileBoundary) { @@ -195,6 +198,7 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) { edit.DeleteFile(4, 700); edit.SetComparatorName("foo"); + edit.SetPersistUserDefinedTimestamps(true); edit.SetLogNumber(kBig + 100); edit.SetNextFile(kBig + 200); edit.SetLastSequence(kBig + 1000); @@ -230,6 +234,7 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) { ASSERT_EQ(3u, new_files[0].second.fd.GetPathId()); ASSERT_EQ(3u, new_files[1].second.fd.GetPathId()); ASSERT_EQ(1u, parsed.GetDeletedFiles().size()); + ASSERT_TRUE(parsed.GetPersistUserDefinedTimestamps()); } TEST_F(VersionEditTest, NewFile4NotSupportedField) { @@ -240,9 +245,10 @@ TEST_F(VersionEditTest, NewFile4NotSupportedField) { kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber, kUnknownOldestAncesterTime, kUnknownFileCreationTime, 300 /* epoch_number */, kUnknownFileChecksum, - kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, true); + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, false); edit.SetComparatorName("foo"); + edit.SetPersistUserDefinedTimestamps(false); edit.SetLogNumber(kBig + 100); edit.SetNextFile(kBig + 200); edit.SetLastSequence(kBig + 1000); @@ -556,7 +562,9 @@ TEST_F(VersionEditTest, AddWalDebug) { std::stringstream ss; ss << "{\"LogNumber\": " << kLogNumbers[i] << ", " << "\"SyncedSizeInBytes\": " << kSizeInBytes[i] << "}"; - if (i < n - 1) ss << ", "; + if (i < n - 1) { + ss << ", "; + } expected_json += ss.str(); } expected_json += "], \"ColumnFamily\": 0}"; diff --git a/db/version_set.cc b/db/version_set.cc index e95e98f79..181c5e356 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -39,6 +39,7 @@ #include "db/version_builder.h" #include "db/version_edit.h" #include "db/version_edit_handler.h" +#include "db/wide/wide_columns_helper.h" #include "file/file_util.h" #include "table/compaction_merging_iterator.h" @@ -121,7 +122,9 @@ Status OverlapWithIterator(const Comparator* ucmp, ParsedInternalKey seek_result; Status s = ParseInternalKey(iter->key(), &seek_result, false /* log_err_key */); // TODO - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } if (ucmp->CompareWithoutTimestamp(seek_result.user_key, largest_user_key) <= 0) { @@ -834,9 +837,9 @@ Version::~Version() { assert(cfd_ != nullptr); uint32_t path_id = f->fd.GetPathId(); assert(path_id < cfd_->ioptions()->cf_paths.size()); - vset_->obsolete_files_.push_back( - ObsoleteFileInfo(f, cfd_->ioptions()->cf_paths[path_id].path, - cfd_->GetFileMetadataCacheReservationManager())); + vset_->obsolete_files_.emplace_back( + f, cfd_->ioptions()->cf_paths[path_id].path, + cfd_->GetFileMetadataCacheReservationManager()); } } } @@ -956,18 +959,21 @@ class LevelIterator final : public InternalIterator { flevel_(flevel), prefix_extractor_(prefix_extractor), file_read_hist_(file_read_hist), - should_sample_(should_sample), caller_(caller), - skip_filters_(skip_filters), - allow_unprepared_value_(allow_unprepared_value), file_index_(flevel_->num_files), - level_(level), range_del_agg_(range_del_agg), pinned_iters_mgr_(nullptr), compaction_boundaries_(compaction_boundaries), - is_next_read_sequential_(false), - block_protection_bytes_per_key_(block_protection_bytes_per_key), range_tombstone_iter_(nullptr), + read_seq_(read_options.snapshot + ? read_options.snapshot->GetSequenceNumber() + : kMaxSequenceNumber), + level_(level), + block_protection_bytes_per_key_(block_protection_bytes_per_key), + should_sample_(should_sample), + skip_filters_(skip_filters), + allow_unprepared_value_(allow_unprepared_value), + is_next_read_sequential_(false), to_return_sentinel_(false) { // Empty level is not supported. assert(flevel_ != nullptr && flevel_->num_files > 0); @@ -1055,6 +1061,10 @@ class LevelIterator final : public InternalIterator { bool IsDeleteRangeSentinelKey() const override { return to_return_sentinel_; } + void SetRangeDelReadSeqno(SequenceNumber read_seq) override { + read_seq_ = read_seq; + } + private: // Return true if at least one invalid file is seen and skipped. bool SkipEmptyFileForward(); @@ -1111,7 +1121,7 @@ class LevelIterator final : public InternalIterator { /*arena=*/nullptr, skip_filters_, level_, /*max_file_size_for_l0_meta_pin=*/0, smallest_compaction_key, largest_compaction_key, allow_unprepared_value_, - block_protection_bytes_per_key_, range_tombstone_iter_); + block_protection_bytes_per_key_, &read_seq_, range_tombstone_iter_); } // Check if current file being fully within iterate_lower_bound. @@ -1141,13 +1151,8 @@ class LevelIterator final : public InternalIterator { const std::shared_ptr& prefix_extractor_; HistogramImpl* file_read_hist_; - bool should_sample_; TableReaderCaller caller_; - bool skip_filters_; - bool allow_unprepared_value_; - bool may_be_out_of_lower_bound_ = true; size_t file_index_; - int level_; RangeDelAggregator* range_del_agg_; IteratorWrapper file_iter_; // May be nullptr PinnedIteratorsManager* pinned_iters_mgr_; @@ -1156,10 +1161,6 @@ class LevelIterator final : public InternalIterator { // tombstones. const std::vector* compaction_boundaries_; - bool is_next_read_sequential_; - - uint8_t block_protection_bytes_per_key_; - // This is set when this level iterator is used under a merging iterator // that processes range tombstones. range_tombstone_iter_ points to where the // merging iterator stores the range tombstones iterator for this level. When @@ -1176,20 +1177,29 @@ class LevelIterator final : public InternalIterator { // *range_tombstone_iter_ points to range tombstones of the current SST file TruncatedRangeDelIterator** range_tombstone_iter_; - // Whether next/prev key is a sentinel key. - bool to_return_sentinel_ = false; // The sentinel key to be returned Slice sentinel_; - // Sets flags for if we should return the sentinel key next. - // The condition for returning sentinel is reaching the end of current - // file_iter_: !Valid() && status.().ok(). - void TrySetDeleteRangeSentinel(const Slice& boundary_key); - void ClearSentinel() { to_return_sentinel_ = false; } + SequenceNumber read_seq_; + int level_; + uint8_t block_protection_bytes_per_key_; + bool should_sample_; + bool skip_filters_; + bool allow_unprepared_value_; + bool may_be_out_of_lower_bound_ = true; + bool is_next_read_sequential_; // Set in Seek() when a prefix seek reaches end of the current file, // and the next file has a different prefix. SkipEmptyFileForward() // will not move to next file when this flag is set. bool prefix_exhausted_ = false; + // Whether next/prev key is a sentinel key. + bool to_return_sentinel_ = false; + + // Sets flags for if we should return the sentinel key next. + // The condition for returning sentinel is reaching the end of current + // file_iter_: !Valid() && status.().ok(). + void TrySetDeleteRangeSentinel(const Slice& boundary_key); + void ClearSentinel() { to_return_sentinel_ = false; } }; void LevelIterator::TrySetDeleteRangeSentinel(const Slice& boundary_key) { @@ -1948,8 +1958,14 @@ double VersionStorageInfo::GetEstimatedCompressionRatioAtLevel( uint64_t sum_file_size_bytes = 0; uint64_t sum_data_size_bytes = 0; for (auto* file_meta : files_[level]) { - sum_file_size_bytes += file_meta->fd.GetFileSize(); - sum_data_size_bytes += file_meta->raw_key_size + file_meta->raw_value_size; + auto raw_size = file_meta->raw_key_size + file_meta->raw_value_size; + // Check if the table property is properly initialized. It might not be + // because in `UpdateAccumulatedStats` we limit the maximum number of + // properties to read once. + if (raw_size > 0) { + sum_file_size_bytes += file_meta->fd.GetFileSize(); + sum_data_size_bytes += raw_size; + } } if (sum_file_size_bytes == 0) { return -1.0; @@ -1999,7 +2015,8 @@ void Version::AddIteratorsForLevel(const ReadOptions& read_options, /*skip_filters=*/false, /*level=*/0, max_file_size_for_l0_meta_pin_, /*smallest_compaction_key=*/nullptr, /*largest_compaction_key=*/nullptr, allow_unprepared_value, - mutable_cf_options_.block_protection_bytes_per_key, &tombstone_iter); + mutable_cf_options_.block_protection_bytes_per_key, + /*range_del_read_seqno=*/nullptr, &tombstone_iter); if (read_options.ignore_range_deletions) { merge_iter_builder->AddIterator(table_iter); } else { @@ -2108,7 +2125,9 @@ VersionStorageInfo::VersionStorageInfo( const Comparator* user_comparator, int levels, CompactionStyle compaction_style, VersionStorageInfo* ref_vstorage, bool _force_consistency_checks, - EpochNumberRequirement epoch_number_requirement) + EpochNumberRequirement epoch_number_requirement, SystemClock* clock, + uint32_t bottommost_file_compaction_delay, + OffpeakTimeOption offpeak_time_option) : internal_comparator_(internal_comparator), user_comparator_(user_comparator), // cfd is nullptr if Version is dummy @@ -2136,9 +2155,12 @@ VersionStorageInfo::VersionStorageInfo( current_num_deletions_(0), current_num_samples_(0), estimated_compaction_needed_bytes_(0), + clock_(clock), + bottommost_file_compaction_delay_(bottommost_file_compaction_delay), finalized_(false), force_consistency_checks_(_force_consistency_checks), - epoch_number_requirement_(epoch_number_requirement) { + epoch_number_requirement_(epoch_number_requirement), + offpeak_time_option_(std::move(offpeak_time_option)) { if (ref_vstorage != nullptr) { accumulated_file_size_ = ref_vstorage->accumulated_file_size_; accumulated_raw_key_size_ = ref_vstorage->accumulated_raw_key_size_; @@ -2180,7 +2202,11 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset, ? nullptr : cfd_->current()->storage_info(), cfd_ == nullptr ? false : cfd_->ioptions()->force_consistency_checks, - epoch_number_requirement), + epoch_number_requirement, + cfd_ == nullptr ? nullptr : cfd_->ioptions()->clock, + cfd_ == nullptr ? 0 + : mutable_cf_options.bottommost_file_compaction_delay, + vset->offpeak_time_option()), vset_(vset), next_(this), prev_(this), @@ -2439,12 +2465,9 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k, fp.GetHitFileLevel()); if (is_blob_index && do_merge && (value || columns)) { - assert(!columns || - (!columns->columns().empty() && - columns->columns().front().name() == kDefaultWideColumnName)); - Slice blob_index = - value ? *value : columns->columns().front().value(); + value ? *value + : WideColumnsHelper::GetDefaultColumn(columns->columns()); TEST_SYNC_POINT_CALLBACK("Version::Get::TamperWithBlobIndex", &blob_index); @@ -2508,21 +2531,16 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k, // merge_operands are in saver and we hit the beginning of the key history // do a final merge of nullptr and operands; if (value || columns) { - std::string result; // `op_failure_scope` (an output parameter) is not provided (set to // nullptr) since a failure must be propagated regardless of its value. *status = MergeHelper::TimedFullMerge( - merge_operator_, user_key, nullptr, merge_context->GetOperands(), - &result, info_log_, db_statistics_, clock_, - /* result_operand */ nullptr, /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); + merge_operator_, user_key, MergeHelper::kNoBaseValue, + merge_context->GetOperands(), info_log_, db_statistics_, clock_, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + value ? value->GetSelf() : nullptr, columns); if (status->ok()) { if (LIKELY(value != nullptr)) { - *(value->GetSelf()) = std::move(result); value->PinSelf(); - } else { - assert(columns != nullptr); - columns->SetPlainValue(std::move(result)); } } } @@ -2759,22 +2777,18 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range, } // merge_operands are in saver and we hit the beginning of the key history // do a final merge of nullptr and operands; - std::string result; - // `op_failure_scope` (an output parameter) is not provided (set to // nullptr) since a failure must be propagated regardless of its value. *status = MergeHelper::TimedFullMerge( - merge_operator_, user_key, nullptr, iter->merge_context.GetOperands(), - &result, info_log_, db_statistics_, clock_, - /* result_operand */ nullptr, /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); + merge_operator_, user_key, MergeHelper::kNoBaseValue, + iter->merge_context.GetOperands(), info_log_, db_statistics_, clock_, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + iter->value ? iter->value->GetSelf() : nullptr, iter->columns); if (LIKELY(iter->value != nullptr)) { - *iter->value->GetSelf() = std::move(result); iter->value->PinSelf(); range->AddValueSize(iter->value->size()); } else { assert(iter->columns); - iter->columns->SetPlainValue(std::move(result)); range->AddValueSize(iter->columns->serialized_size()); } @@ -3053,9 +3067,7 @@ void VersionStorageInfo::PrepareForVersionAppend( GenerateFileIndexer(); GenerateLevelFilesBrief(); GenerateLevel0NonOverlapping(); - if (!immutable_options.allow_ingest_behind) { - GenerateBottommostFiles(); - } + GenerateBottommostFiles(); GenerateFileLocationIndex(); } @@ -3088,7 +3100,9 @@ bool Version::MaybeInitializeFileMetaData(const ReadOptions& read_options, file_meta->fd.GetNumber(), s.ToString().c_str()); return false; } - if (tp.get() == nullptr) return false; + if (tp.get() == nullptr) { + return false; + } file_meta->num_entries = tp->num_entries; file_meta->num_deletions = tp->num_deletions; file_meta->raw_value_size = tp->raw_value_size; @@ -3571,26 +3585,16 @@ void VersionStorageInfo::ComputeCompactionScore( } } ComputeFilesMarkedForCompaction(max_output_level); - if (!immutable_options.allow_ingest_behind) { - ComputeBottommostFilesMarkedForCompaction(); - } - if (mutable_cf_options.ttl > 0 && - compaction_style_ == kCompactionStyleLevel) { - ComputeExpiredTtlFiles(immutable_options, mutable_cf_options.ttl); - } - if (mutable_cf_options.periodic_compaction_seconds > 0) { - ComputeFilesMarkedForPeriodicCompaction( - immutable_options, mutable_cf_options.periodic_compaction_seconds, - max_output_level); - } - - if (mutable_cf_options.enable_blob_garbage_collection && - mutable_cf_options.blob_garbage_collection_age_cutoff > 0.0 && - mutable_cf_options.blob_garbage_collection_force_threshold < 1.0) { - ComputeFilesMarkedForForcedBlobGC( - mutable_cf_options.blob_garbage_collection_age_cutoff, - mutable_cf_options.blob_garbage_collection_force_threshold); - } + ComputeBottommostFilesMarkedForCompaction( + immutable_options.allow_ingest_behind); + ComputeExpiredTtlFiles(immutable_options, mutable_cf_options.ttl); + ComputeFilesMarkedForPeriodicCompaction( + immutable_options, mutable_cf_options.periodic_compaction_seconds, + max_output_level); + ComputeFilesMarkedForForcedBlobGC( + mutable_cf_options.blob_garbage_collection_age_cutoff, + mutable_cf_options.blob_garbage_collection_force_threshold, + mutable_cf_options.enable_blob_garbage_collection); EstimateCompactionBytesNeeded(mutable_cf_options); } @@ -3620,9 +3624,10 @@ void VersionStorageInfo::ComputeFilesMarkedForCompaction(int last_level) { void VersionStorageInfo::ComputeExpiredTtlFiles( const ImmutableOptions& ioptions, const uint64_t ttl) { - assert(ttl > 0); - expired_ttl_files_.clear(); + if (ttl == 0 || compaction_style_ != CompactionStyle::kCompactionStyleLevel) { + return; + } int64_t _current_time; auto status = ioptions.clock->GetCurrentTime(&_current_time); @@ -3647,9 +3652,10 @@ void VersionStorageInfo::ComputeExpiredTtlFiles( void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( const ImmutableOptions& ioptions, const uint64_t periodic_compaction_seconds, int last_level) { - assert(periodic_compaction_seconds > 0); - files_marked_for_periodic_compaction_.clear(); + if (periodic_compaction_seconds == 0) { + return; + } int64_t temp_current_time; auto status = ioptions.clock->GetCurrentTime(&temp_current_time); @@ -3667,6 +3673,16 @@ void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( const uint64_t allowed_time_limit = current_time - periodic_compaction_seconds; + // Find the adjust_allowed_time_limit such that it includes files that are + // going to expire by the time next daily offpeak starts. + const OffpeakTimeInfo offpeak_time_info = + offpeak_time_option_.GetOffpeakTimeInfo(current_time); + const uint64_t adjusted_allowed_time_limit = + allowed_time_limit + + (offpeak_time_info.is_now_offpeak + ? offpeak_time_info.seconds_till_next_offpeak_start + : 0); + for (int level = 0; level <= last_level; level++) { for (auto f : files_[level]) { if (!f->being_compacted) { @@ -3693,7 +3709,7 @@ void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( } } if (file_modification_time > 0 && - file_modification_time < allowed_time_limit) { + file_modification_time < adjusted_allowed_time_limit) { files_marked_for_periodic_compaction_.emplace_back(level, f); } } @@ -3703,8 +3719,14 @@ void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( void VersionStorageInfo::ComputeFilesMarkedForForcedBlobGC( double blob_garbage_collection_age_cutoff, - double blob_garbage_collection_force_threshold) { + double blob_garbage_collection_force_threshold, + bool enable_blob_garbage_collection) { files_marked_for_forced_blob_gc_.clear(); + if (!(enable_blob_garbage_collection && + blob_garbage_collection_age_cutoff > 0.0 && + blob_garbage_collection_force_threshold < 1.0)) { + return; + } if (blob_files_.empty()) { return; @@ -4161,25 +4183,64 @@ void VersionStorageInfo::GenerateFileLocationIndex() { } } -void VersionStorageInfo::UpdateOldestSnapshot(SequenceNumber seqnum) { +void VersionStorageInfo::UpdateOldestSnapshot(SequenceNumber seqnum, + bool allow_ingest_behind) { assert(seqnum >= oldest_snapshot_seqnum_); oldest_snapshot_seqnum_ = seqnum; if (oldest_snapshot_seqnum_ > bottommost_files_mark_threshold_) { - ComputeBottommostFilesMarkedForCompaction(); + ComputeBottommostFilesMarkedForCompaction(allow_ingest_behind); } } -void VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction() { +void VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction( + bool allow_ingest_behind) { bottommost_files_marked_for_compaction_.clear(); bottommost_files_mark_threshold_ = kMaxSequenceNumber; + if (allow_ingest_behind) { + return; + } + // If a file's creation time is larger than creation_time_ub, + // it is too new to be marked for compaction. + int64_t creation_time_ub = 0; + bool needs_delay = bottommost_file_compaction_delay_ > 0; + if (needs_delay) { + int64_t current_time = 0; + clock_->GetCurrentTime(¤t_time).PermitUncheckedError(); + // Note that if GetCurrentTime() fails, current_time will be 0. + // We will treat it as is and treat all files as too new. + // The subtraction will not underflow since + // bottommost_file_compaction_delay_ is of type uint32_t. + creation_time_ub = + current_time - static_cast(bottommost_file_compaction_delay_); + } + for (auto& level_and_file : bottommost_files_) { if (!level_and_file.second->being_compacted && level_and_file.second->fd.largest_seqno != 0) { // largest_seqno might be nonzero due to containing the final key in an - // earlier compaction, whose seqnum we didn't zero out. Multiple deletions - // ensures the file really contains deleted or overwritten keys. + // earlier compaction, whose seqnum we didn't zero out. if (level_and_file.second->fd.largest_seqno < oldest_snapshot_seqnum_) { - bottommost_files_marked_for_compaction_.push_back(level_and_file); + if (!needs_delay) { + bottommost_files_marked_for_compaction_.push_back(level_and_file); + } else if (creation_time_ub > 0) { + int64_t creation_time = static_cast( + level_and_file.second->TryGetFileCreationTime()); + if (creation_time == kUnknownFileCreationTime || + creation_time <= creation_time_ub) { + bottommost_files_marked_for_compaction_.push_back(level_and_file); + } else { + // Just ignore this file for both + // bottommost_files_marked_for_compaction_ and + // bottommost_files_mark_threshold_. The next time + // this method is called, it will try this file again. The method + // is called after a new Version creation (compaction, flush, etc.), + // after a compaction is picked, and after a snapshot newer than + // bottommost_files_mark_threshold_ is released. + } + } else { + // creation_time_ub <= 0, all files are too new to be marked for + // compaction. + } } else { bottommost_files_mark_threshold_ = std::min(bottommost_files_mark_threshold_, @@ -4455,7 +4516,9 @@ const char* VersionStorageInfo::LevelSummary( for (int i = 0; i < num_levels(); i++) { int sz = sizeof(scratch->buffer) - len; int ret = snprintf(scratch->buffer + len, sz, "%d ", int(files_[i].size())); - if (ret < 0 || ret >= sz) break; + if (ret < 0 || ret >= sz) { + break; + } len += ret; } if (len > 0) { @@ -4485,7 +4548,9 @@ const char* VersionStorageInfo::LevelFileSummary(FileSummaryStorage* scratch, "#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ", f->fd.GetNumber(), f->fd.smallest_seqno, sztxt, static_cast(f->being_compacted)); - if (ret < 0 || ret >= sz) break; + if (ret < 0 || ret >= sz) { + break; + } len += ret; } // overwrite the last space (only if files_[level].size() is non-zero) @@ -4517,25 +4582,27 @@ uint64_t VersionStorageInfo::GetMaxEpochNumberOfFiles() const { return max_epoch_number; } -void VersionStorageInfo::RecoverEpochNumbers(ColumnFamilyData* cfd) { - cfd->ResetNextEpochNumber(); +void VersionStorageInfo::RecoverEpochNumbers(ColumnFamilyData* cfd, + bool restart_epoch, bool force) { + if (restart_epoch) { + cfd->ResetNextEpochNumber(); - bool reserve_epoch_num_for_file_ingested_behind = - cfd->ioptions()->allow_ingest_behind; - if (reserve_epoch_num_for_file_ingested_behind) { - uint64_t reserved_epoch_number = cfd->NewEpochNumber(); - assert(reserved_epoch_number == kReservedEpochNumberForFileIngestedBehind); - ROCKS_LOG_INFO(cfd->ioptions()->info_log.get(), - "[%s]CF has reserved epoch number %" PRIu64 - " for files ingested " - "behind since `Options::allow_ingest_behind` is true", - cfd->GetName().c_str(), reserved_epoch_number); + bool reserve_epoch_num_for_file_ingested_behind = + cfd->ioptions()->allow_ingest_behind; + if (reserve_epoch_num_for_file_ingested_behind) { + uint64_t reserved_epoch_number = cfd->NewEpochNumber(); + assert(reserved_epoch_number == + kReservedEpochNumberForFileIngestedBehind); + ROCKS_LOG_INFO(cfd->ioptions()->info_log.get(), + "[%s]CF has reserved epoch number %" PRIu64 + " for files ingested " + "behind since `Options::allow_ingest_behind` is true", + cfd->GetName().c_str(), reserved_epoch_number); + } } - if (HasMissingEpochNumber()) { - assert(epoch_number_requirement_ == EpochNumberRequirement::kMightMissing); - assert(num_levels_ >= 1); - + bool missing_epoch_number = HasMissingEpochNumber(); + if (missing_epoch_number || force) { for (int level = num_levels_ - 1; level >= 1; --level) { auto& files_at_level = files_[level]; if (files_at_level.empty()) { @@ -4546,17 +4613,19 @@ void VersionStorageInfo::RecoverEpochNumbers(ColumnFamilyData* cfd) { f->epoch_number = next_epoch_number; } } - for (auto file_meta_iter = files_[0].rbegin(); file_meta_iter != files_[0].rend(); file_meta_iter++) { FileMetaData* f = *file_meta_iter; f->epoch_number = cfd->NewEpochNumber(); } - - ROCKS_LOG_WARN(cfd->ioptions()->info_log.get(), - "[%s]CF's epoch numbers are inferred based on seqno", - cfd->GetName().c_str()); - epoch_number_requirement_ = EpochNumberRequirement::kMustPresent; + if (missing_epoch_number) { + assert(epoch_number_requirement_ == + EpochNumberRequirement::kMightMissing); + ROCKS_LOG_WARN(cfd->ioptions()->info_log.get(), + "[%s]CF's epoch numbers are inferred based on seqno", + cfd->GetName().c_str()); + epoch_number_requirement_ = EpochNumberRequirement::kMustPresent; + } } else { assert(epoch_number_requirement_ == EpochNumberRequirement::kMustPresent); cfd->SetNextEpochNumber( @@ -4952,15 +5021,15 @@ struct VersionSet::ManifestWriter { Status AtomicGroupReadBuffer::AddEdit(VersionEdit* edit) { assert(edit); - if (edit->is_in_atomic_group_) { + if (edit->IsInAtomicGroup()) { TEST_SYNC_POINT("AtomicGroupReadBuffer::AddEdit:AtomicGroup"); if (replay_buffer_.empty()) { - replay_buffer_.resize(edit->remaining_entries_ + 1); + replay_buffer_.resize(edit->GetRemainingEntries() + 1); TEST_SYNC_POINT_CALLBACK( "AtomicGroupReadBuffer::AddEdit:FirstInAtomicGroup", edit); } read_edits_in_atomic_group_++; - if (read_edits_in_atomic_group_ + edit->remaining_entries_ != + if (read_edits_in_atomic_group_ + edit->GetRemainingEntries() != static_cast(replay_buffer_.size())) { TEST_SYNC_POINT_CALLBACK( "AtomicGroupReadBuffer::AddEdit:IncorrectAtomicGroupSize", edit); @@ -4995,15 +5064,14 @@ void AtomicGroupReadBuffer::Clear() { replay_buffer_.clear(); } -VersionSet::VersionSet(const std::string& dbname, - const ImmutableDBOptions* _db_options, - const FileOptions& storage_options, Cache* table_cache, - WriteBufferManager* write_buffer_manager, - WriteController* write_controller, - BlockCacheTracer* const block_cache_tracer, - const std::shared_ptr& io_tracer, - const std::string& db_id, - const std::string& db_session_id) +VersionSet::VersionSet( + const std::string& dbname, const ImmutableDBOptions* _db_options, + const FileOptions& storage_options, Cache* table_cache, + WriteBufferManager* write_buffer_manager, WriteController* write_controller, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, const std::string& db_id, + const std::string& db_session_id, const std::string& daily_offpeak_time_utc, + ErrorHandler* const error_handler) : column_family_set_(new ColumnFamilySet( dbname, _db_options, storage_options, table_cache, write_buffer_manager, write_controller, block_cache_tracer, io_tracer, @@ -5028,7 +5096,9 @@ VersionSet::VersionSet(const std::string& dbname, file_options_(storage_options), block_cache_tracer_(block_cache_tracer), io_tracer_(io_tracer), - db_session_id_(db_session_id) {} + db_session_id_(db_session_id), + offpeak_time_option_(OffpeakTimeOption(daily_offpeak_time_utc)), + error_handler_(error_handler) {} VersionSet::~VersionSet() { // we need to delete column_family_set_ because its destructor depends on @@ -5127,6 +5197,8 @@ Status VersionSet::ProcessManifestWrites( autovector versions; autovector mutable_cf_options_ptrs; std::vector> builder_guards; + autovector*> files_to_quarantine_if_commit_fail; + autovector limbo_descriptor_log_file_number; // Tracking `max_last_sequence` is needed to ensure we write // `VersionEdit::last_sequence_`s in non-decreasing order according to the @@ -5160,15 +5232,15 @@ Status VersionSet::ProcessManifestWrites( // don't update, then Recover can report corrupted atomic group because // the `remaining_entries_` do not match. if (!batch_edits.empty()) { - if (batch_edits.back()->is_in_atomic_group_ && - batch_edits.back()->remaining_entries_ > 0) { + if (batch_edits.back()->IsInAtomicGroup() && + batch_edits.back()->GetRemainingEntries() > 0) { assert(group_start < batch_edits.size()); const auto& edit_list = last_writer->edit_list; size_t k = 0; while (k < edit_list.size()) { - if (!edit_list[k]->is_in_atomic_group_) { + if (!edit_list[k]->IsInAtomicGroup()) { break; - } else if (edit_list[k]->remaining_entries_ == 0) { + } else if (edit_list[k]->GetRemainingEntries() == 0) { ++k; break; } @@ -5176,8 +5248,10 @@ Status VersionSet::ProcessManifestWrites( } for (auto i = group_start; i < batch_edits.size(); ++i) { assert(static_cast(k) <= - batch_edits.back()->remaining_entries_); - batch_edits[i]->remaining_entries_ -= static_cast(k); + batch_edits.back()->GetRemainingEntries()); + batch_edits[i]->SetRemainingEntries( + batch_edits[i]->GetRemainingEntries() - + static_cast(k)); } } } @@ -5220,10 +5294,10 @@ Status VersionSet::ProcessManifestWrites( assert(ucmp); std::optional edit_ts_sz = ucmp->timestamp_size(); for (const auto& e : last_writer->edit_list) { - if (e->is_in_atomic_group_) { - if (batch_edits.empty() || !batch_edits.back()->is_in_atomic_group_ || - (batch_edits.back()->is_in_atomic_group_ && - batch_edits.back()->remaining_entries_ == 0)) { + if (e->IsInAtomicGroup()) { + if (batch_edits.empty() || !batch_edits.back()->IsInAtomicGroup() || + (batch_edits.back()->IsInAtomicGroup() && + batch_edits.back()->GetRemainingEntries() == 0)) { group_start = batch_edits.size(); } } else if (group_start != std::numeric_limits::max()) { @@ -5262,7 +5336,7 @@ Status VersionSet::ProcessManifestWrites( // remaining_entries_. size_t k = 0; while (k < batch_edits.size()) { - while (k < batch_edits.size() && !batch_edits[k]->is_in_atomic_group_) { + while (k < batch_edits.size() && !batch_edits[k]->IsInAtomicGroup()) { ++k; } if (k == batch_edits.size()) { @@ -5270,19 +5344,19 @@ Status VersionSet::ProcessManifestWrites( } size_t i = k; while (i < batch_edits.size()) { - if (!batch_edits[i]->is_in_atomic_group_) { + if (!batch_edits[i]->IsInAtomicGroup()) { break; } - assert(i - k + batch_edits[i]->remaining_entries_ == - batch_edits[k]->remaining_entries_); - if (batch_edits[i]->remaining_entries_ == 0) { + assert(i - k + batch_edits[i]->GetRemainingEntries() == + batch_edits[k]->GetRemainingEntries()); + if (batch_edits[i]->GetRemainingEntries() == 0) { ++i; break; } ++i; } - assert(batch_edits[i - 1]->is_in_atomic_group_); - assert(0 == batch_edits[i - 1]->remaining_entries_); + assert(batch_edits[i - 1]->IsInAtomicGroup()); + assert(0 == batch_edits[i - 1]->GetRemainingEntries()); std::vector tmp; for (size_t j = k; j != i; ++j) { tmp.emplace_back(batch_edits[j]); @@ -5319,9 +5393,9 @@ Status VersionSet::ProcessManifestWrites( } for (const auto* cfd : *column_family_set_) { assert(curr_state.find(cfd->GetID()) == curr_state.end()); - curr_state.emplace(std::make_pair( + curr_state.emplace( cfd->GetID(), - MutableCFState(cfd->GetLogNumber(), cfd->GetFullHistoryTsLow()))); + MutableCFState(cfd->GetLogNumber(), cfd->GetFullHistoryTsLow())); } for (const auto& wal : wals_.GetWals()) { @@ -5408,6 +5482,8 @@ Status VersionSet::ProcessManifestWrites( assert(batch_edits.size() == batch_edits_ts_sz.size()); for (size_t bidx = 0; bidx < batch_edits.size(); bidx++) { auto& e = batch_edits[bidx]; + files_to_quarantine_if_commit_fail.push_back( + e->GetFilesToQuarantineIfCommitFail()); std::string record; if (!e->EncodeTo(&record, batch_edits_ts_sz[bidx])) { s = Status::Corruption("Unable to encode VersionEdit:" + @@ -5457,6 +5533,11 @@ Status VersionSet::ProcessManifestWrites( dir_contains_current_file); if (!io_s.ok()) { s = io_s; + // Quarantine old manifest file in case new manifest file's CURRENT file + // wasn't created successfully and the old manifest is needed. + limbo_descriptor_log_file_number.push_back(manifest_file_number_); + files_to_quarantine_if_commit_fail.push_back( + &limbo_descriptor_log_file_number); } } @@ -5465,7 +5546,7 @@ Status VersionSet::ProcessManifestWrites( new_manifest_file_size = descriptor_log_->file()->GetFileSize(); } - if (first_writer.edit_list.front()->is_column_family_drop_) { + if (first_writer.edit_list.front()->IsColumnFamilyDrop()) { TEST_SYNC_POINT("VersionSet::LogAndApply::ColumnFamilyDrop:0"); TEST_SYNC_POINT("VersionSet::LogAndApply::ColumnFamilyDrop:1"); TEST_SYNC_POINT("VersionSet::LogAndApply::ColumnFamilyDrop:2"); @@ -5493,9 +5574,16 @@ Status VersionSet::ProcessManifestWrites( if (!io_s.ok()) { if (io_status_.ok()) { io_status_ = io_s; + if (error_handler_) { + error_handler_->AddFilesToQuarantine( + files_to_quarantine_if_commit_fail); + } } } else if (!io_status_.ok()) { io_status_ = io_s; + if (error_handler_) { + error_handler_->ClearFilesToQuarantine(); + } } // Append the old manifest file to the obsolete_manifest_ list to be deleted @@ -5507,13 +5595,13 @@ Status VersionSet::ProcessManifestWrites( // Install the new versions if (s.ok()) { - if (first_writer.edit_list.front()->is_column_family_add_) { + if (first_writer.edit_list.front()->IsColumnFamilyAdd()) { assert(batch_edits.size() == 1); assert(new_cf_options != nullptr); assert(max_last_sequence == descriptor_last_sequence_); CreateColumnFamily(*new_cf_options, read_options, first_writer.edit_list.front()); - } else if (first_writer.edit_list.front()->is_column_family_drop_) { + } else if (first_writer.edit_list.front()->IsColumnFamilyDrop()) { assert(batch_edits.size() == 1); assert(max_last_sequence == descriptor_last_sequence_); first_writer.cfd->SetDropped(); @@ -5526,22 +5614,22 @@ Status VersionSet::ProcessManifestWrites( for (const auto& e : batch_edits) { ColumnFamilyData* cfd = nullptr; if (!e->IsColumnFamilyManipulation()) { - cfd = column_family_set_->GetColumnFamily(e->column_family_); + cfd = column_family_set_->GetColumnFamily(e->GetColumnFamily()); // e would not have been added to batch_edits if its corresponding // column family is dropped. assert(cfd); } if (cfd) { - if (e->has_log_number_ && e->log_number_ > cfd->GetLogNumber()) { - cfd->SetLogNumber(e->log_number_); + if (e->HasLogNumber() && e->GetLogNumber() > cfd->GetLogNumber()) { + cfd->SetLogNumber(e->GetLogNumber()); } if (e->HasFullHistoryTsLow()) { cfd->SetFullHistoryTsLow(e->GetFullHistoryTsLow()); } } - if (e->has_min_log_number_to_keep_) { + if (e->HasMinLogNumberToKeep()) { last_min_log_number_to_keep = - std::max(last_min_log_number_to_keep, e->min_log_number_to_keep_); + std::max(last_min_log_number_to_keep, e->GetMinLogNumberToKeep()); } } @@ -5558,7 +5646,7 @@ Status VersionSet::ProcessManifestWrites( descriptor_last_sequence_ = max_last_sequence; manifest_file_number_ = pending_manifest_file_number_; manifest_file_size_ = new_manifest_file_size; - prev_log_number_ = first_writer.edit_list.front()->prev_log_number_; + prev_log_number_ = first_writer.edit_list.front()->GetPrevLogNumber(); } else { std::string version_edits; for (auto& e : batch_edits) { @@ -5706,7 +5794,7 @@ Status VersionSet::LogAndApply( int num_cfds = static_cast(column_family_datas.size()); if (num_cfds == 1 && column_family_datas[0] == nullptr) { assert(edit_lists.size() == 1 && edit_lists[0].size() == 1); - assert(edit_lists[0][0]->is_column_family_add_); + assert(edit_lists[0][0]->IsColumnFamilyAdd()); assert(new_cf_options != nullptr); } std::deque writers; @@ -5770,7 +5858,7 @@ void VersionSet::LogAndApplyCFHelper(VersionEdit* edit, edit->SetNextFile(next_file_number_.load()); assert(!edit->HasLastSequence()); edit->SetLastSequence(*max_last_sequence); - if (edit->is_column_family_drop_) { + if (edit->IsColumnFamilyDrop()) { // if we drop column family, we have to make sure to save max column family, // so that we don't reuse existing ID edit->SetMaxColumnFamily(column_family_set_->GetMaxColumnFamily()); @@ -5788,12 +5876,12 @@ Status VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd, assert(!edit->IsColumnFamilyManipulation()); assert(max_last_sequence != nullptr); - if (edit->has_log_number_) { - assert(edit->log_number_ >= cfd->GetLogNumber()); - assert(edit->log_number_ < next_file_number_.load()); + if (edit->HasLogNumber()) { + assert(edit->GetLogNumber() >= cfd->GetLogNumber()); + assert(edit->GetLogNumber() < next_file_number_.load()); } - if (!edit->has_prev_log_number_) { + if (!edit->HasPrevLogNumber()) { edit->SetPrevLogNumber(prev_log_number_); } edit->SetNextFile(next_file_number_.load()); @@ -5885,7 +5973,7 @@ Status VersionSet::Recover( handler.Iterate(reader, &log_read_status); s = handler.status(); if (s.ok()) { - log_number = handler.GetVersionEditParams().log_number_; + log_number = handler.GetVersionEditParams().GetLogNumber(); current_manifest_file_size = reader.GetReadOffset(); assert(current_manifest_file_size != 0); handler.GetDbId(db_id); @@ -6153,7 +6241,8 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname, VersionSet versions(dbname, &db_options, file_options, tc.get(), &wb, &wc, nullptr /*BlockCacheTracer*/, nullptr /*IOTracer*/, /*db_id*/ "", - /*db_session_id*/ ""); + /*db_session_id*/ "", options->daily_offpeak_time_utc, + /*error_handler_*/ nullptr); Status status; std::vector dummy; @@ -6448,6 +6537,8 @@ Status VersionSet::WriteCurrentStateToManifest( } edit.SetComparatorName( cfd->internal_comparator().user_comparator()->Name()); + edit.SetPersistUserDefinedTimestamps( + cfd->ioptions()->persist_user_defined_timestamps); std::string record; if (!edit.EncodeTo(&record)) { return Status::Corruption("Unable to Encode VersionEdit:" + @@ -6909,6 +7000,7 @@ InternalIterator* VersionSet::MakeInputIterator( /*largest_compaction_key=*/nullptr, /*allow_unprepared_value=*/false, c->mutable_cf_options()->block_protection_bytes_per_key, + /*range_del_read_seqno=*/nullptr, /*range_del_iter=*/&range_tombstone_iter); range_tombstones.emplace_back(range_tombstone_iter, nullptr); } @@ -7053,7 +7145,7 @@ uint64_t VersionSet::GetObsoleteSstFilesSize() const { ColumnFamilyData* VersionSet::CreateColumnFamily( const ColumnFamilyOptions& cf_options, const ReadOptions& read_options, const VersionEdit* edit) { - assert(edit->is_column_family_add_); + assert(edit->IsColumnFamilyAdd()); MutableCFOptions dummy_cf_options; Version* dummy_versions = @@ -7062,7 +7154,7 @@ ColumnFamilyData* VersionSet::CreateColumnFamily( // by avoiding calling "delete" explicitly (~Version is private) dummy_versions->Ref(); auto new_cfd = column_family_set_->CreateColumnFamily( - edit->column_family_name_, edit->column_family_, dummy_versions, + edit->GetColumnFamilyName(), edit->GetColumnFamily(), dummy_versions, cf_options); Version* v = new Version(new_cfd, this, file_options_, @@ -7079,7 +7171,7 @@ ColumnFamilyData* VersionSet::CreateColumnFamily( // cfd is not available to client new_cfd->CreateNewMemtable(*new_cfd->GetLatestMutableCFOptions(), LastSequence()); - new_cfd->SetLogNumber(edit->log_number_); + new_cfd->SetLogNumber(edit->GetLogNumber()); return new_cfd; } @@ -7191,9 +7283,10 @@ ReactiveVersionSet::ReactiveVersionSet( : VersionSet(dbname, _db_options, _file_options, table_cache, write_buffer_manager, write_controller, /*block_cache_tracer=*/nullptr, io_tracer, /*db_id*/ "", - /*db_session_id*/ "") {} + /*db_session_id*/ "", /*daily_offpeak_time_utc*/ "", + /*error_handler=*/nullptr) {} -ReactiveVersionSet::~ReactiveVersionSet() {} +ReactiveVersionSet::~ReactiveVersionSet() = default; Status ReactiveVersionSet::Recover( const std::vector& column_families, diff --git a/db/version_set.h b/db/version_set.h index e32d0ff11..1ef256a7b 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -38,6 +38,7 @@ #include "db/compaction/compaction.h" #include "db/compaction/compaction_picker.h" #include "db/dbformat.h" +#include "db/error_handler.h" #include "db/file_indexer.h" #include "db/log_reader.h" #include "db/range_del_aggregator.h" @@ -53,6 +54,7 @@ #endif #include "monitoring/instrumented_mutex.h" #include "options/db_options.h" +#include "options/offpeak_time_info.h" #include "port/port.h" #include "rocksdb/env.h" #include "rocksdb/file_checksum.h" @@ -132,8 +134,10 @@ class VersionStorageInfo { CompactionStyle compaction_style, VersionStorageInfo* src_vstorage, bool _force_consistency_checks, - EpochNumberRequirement epoch_number_requirement = - EpochNumberRequirement::kMustPresent); + EpochNumberRequirement epoch_number_requirement, + SystemClock* clock, + uint32_t bottommost_file_compaction_delay, + OffpeakTimeOption offpeak_time_option); // No copying allowed VersionStorageInfo(const VersionStorageInfo&) = delete; void operator=(const VersionStorageInfo&) = delete; @@ -227,7 +231,7 @@ class VersionStorageInfo { // eligible for compaction. // // REQUIRES: DB mutex held - void ComputeBottommostFilesMarkedForCompaction(); + void ComputeBottommostFilesMarkedForCompaction(bool allow_ingest_behind); // This computes files_marked_for_forced_blob_gc_ and is called by // ComputeCompactionScore() @@ -235,14 +239,16 @@ class VersionStorageInfo { // REQUIRES: DB mutex held void ComputeFilesMarkedForForcedBlobGC( double blob_garbage_collection_age_cutoff, - double blob_garbage_collection_force_threshold); + double blob_garbage_collection_force_threshold, + bool enable_blob_garbage_collection); bool level0_non_overlapping() const { return level0_non_overlapping_; } // Updates the oldest snapshot and related internal state, like the bottommost // files marked for compaction. // REQUIRES: DB mutex held - void UpdateOldestSnapshot(SequenceNumber oldest_snapshot_seqnum); + void UpdateOldestSnapshot(SequenceNumber oldest_snapshot_seqnum, + bool allow_ingest_behind); int MaxInputLevel() const; int MaxOutputLevel(bool allow_ingest_behind) const; @@ -335,7 +341,15 @@ class VersionStorageInfo { EpochNumberRequirement epoch_number_requirement) { epoch_number_requirement_ = epoch_number_requirement; } - void RecoverEpochNumbers(ColumnFamilyData* cfd); + // Ensure all files have epoch number set. + // If there is a file missing epoch number, all files' epoch number will be + // reset according to CF's epoch number. Otherwise, the CF will be updated + // with the max epoch number of the files. + // + // @param restart_epoch This CF's epoch number will be reset to start from 0. + // @param force Force resetting all files' epoch number. + void RecoverEpochNumbers(ColumnFamilyData* cfd, bool restart_epoch = true, + bool force = false); class FileLocation { public: @@ -484,6 +498,12 @@ class VersionStorageInfo { files_marked_for_periodic_compaction_.emplace_back(level, f); } + // REQUIRES: PrepareForVersionAppend has been called + const autovector>& BottommostFiles() const { + assert(finalized_); + return bottommost_files_; + } + // REQUIRES: ComputeCompactionScore has been called // REQUIRES: DB mutex held during access const autovector>& @@ -748,6 +768,11 @@ class VersionStorageInfo { // target sizes. uint64_t estimated_compaction_needed_bytes_; + // Used for computing bottommost files marked for compaction and checking for + // offpeak time. + SystemClock* clock_; + uint32_t bottommost_file_compaction_delay_; + bool finalized_; // If set to true, we will run consistency checks even if RocksDB @@ -756,6 +781,8 @@ class VersionStorageInfo { EpochNumberRequirement epoch_number_requirement_; + OffpeakTimeOption offpeak_time_option_; + friend class Version; friend class VersionSet; }; @@ -1139,7 +1166,9 @@ class VersionSet { WriteController* write_controller, BlockCacheTracer* const block_cache_tracer, const std::shared_ptr& io_tracer, - const std::string& db_id, const std::string& db_session_id); + const std::string& db_id, const std::string& db_session_id, + const std::string& daily_offpeak_time_utc, + ErrorHandler* const error_handler); // No copying allowed VersionSet(const VersionSet&) = delete; void operator=(const VersionSet&) = delete; @@ -1169,7 +1198,8 @@ class VersionSet { const MutableCFOptions& mutable_cf_options, const ReadOptions& read_options, VersionEdit* edit, InstrumentedMutex* mu, FSDirectory* dir_contains_current_file, bool new_descriptor_log = false, - const ColumnFamilyOptions* column_family_options = nullptr) { + const ColumnFamilyOptions* column_family_options = nullptr, + const std::function& manifest_wcb = {}) { autovector cfds; cfds.emplace_back(column_family_data); autovector mutable_cf_options_list; @@ -1180,7 +1210,7 @@ class VersionSet { edit_lists.emplace_back(edit_list); return LogAndApply(cfds, mutable_cf_options_list, read_options, edit_lists, mu, dir_contains_current_file, new_descriptor_log, - column_family_options); + column_family_options, {manifest_wcb}); } // The batch version. If edit_list.size() > 1, caller must ensure that // no edit in the list column family add or drop @@ -1493,6 +1523,14 @@ class VersionSet { new_options.writable_file_max_buffer_size; } + // TODO - Consider updating together when file options change in SetDBOptions + const OffpeakTimeOption& offpeak_time_option() { + return offpeak_time_option_; + } + void ChangeOffpeakTimeOption(const std::string& daily_offpeak_time_utc) { + offpeak_time_option_.SetFromOffpeakTimeString(daily_offpeak_time_utc); + } + const ImmutableDBOptions* db_options() const { return db_options_; } static uint64_t GetNumLiveVersions(Version* dummy_versions); @@ -1521,9 +1559,6 @@ class VersionSet { } protected: - using VersionBuilderMap = - UnorderedMap>; - struct ManifestWriter; friend class Version; @@ -1646,6 +1681,12 @@ class VersionSet { std::string db_session_id_; + // Off-peak time option used for compaction scoring + OffpeakTimeOption offpeak_time_option_; + + // Pointer to the DB's ErrorHandler. + ErrorHandler* const error_handler_; + private: // REQUIRES db mutex at beginning. may release and re-acquire db mutex Status ProcessManifestWrites(std::deque& writers, diff --git a/db/version_set_sync_and_async.h b/db/version_set_sync_and_async.h index 2507762e8..75776b620 100644 --- a/db/version_set_sync_and_async.h +++ b/db/version_set_sync_and_async.h @@ -113,12 +113,9 @@ DEFINE_SYNC_AND_ASYNC(Status, Version::MultiGetFromSST) } else { assert(iter->columns); - assert(!iter->columns->columns().empty()); - assert(iter->columns->columns().front().name() == - kDefaultWideColumnName); - tmp_s = - blob_index.DecodeFrom(iter->columns->columns().front().value()); + tmp_s = blob_index.DecodeFrom( + WideColumnsHelper::GetDefaultColumn(iter->columns->columns())); } if (tmp_s.ok()) { diff --git a/db/version_set_test.cc b/db/version_set_test.cc index 1d5971c59..5def22925 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -21,6 +21,7 @@ #include "table/block_based/block_based_table_factory.h" #include "table/mock_table.h" #include "table/unique_id_impl.h" +#include "test_util/mock_time_env.h" #include "test_util/testharness.h" #include "test_util/testutil.h" #include "util/string_util.h" @@ -33,7 +34,7 @@ class GenerateLevelFilesBriefTest : public testing::Test { LevelFilesBrief file_level_; Arena arena_; - GenerateLevelFilesBriefTest() {} + GenerateLevelFilesBriefTest() = default; ~GenerateLevelFilesBriefTest() override { for (size_t i = 0; i < files_.size(); i++) { @@ -130,7 +131,10 @@ class VersionStorageInfoTestBase : public testing::Test { mutable_cf_options_(options_), vstorage_(&icmp_, ucmp_, 6, kCompactionStyleLevel, /*src_vstorage=*/nullptr, - /*_force_consistency_checks=*/false) {} + /*_force_consistency_checks=*/false, + EpochNumberRequirement::kMustPresent, ioptions_.clock, + mutable_cf_options_.bottommost_file_compaction_delay, + OffpeakTimeOption()) {} ~VersionStorageInfoTestBase() override { for (int i = 0; i < vstorage_.num_levels(); ++i) { @@ -209,7 +213,7 @@ class VersionStorageInfoTest : public VersionStorageInfoTestBase { public: VersionStorageInfoTest() : VersionStorageInfoTestBase(BytewiseComparator()) {} - ~VersionStorageInfoTest() override {} + ~VersionStorageInfoTest() override = default; }; TEST_F(VersionStorageInfoTest, MaxBytesForLevelStatic) { @@ -516,6 +520,55 @@ TEST_F(VersionStorageInfoTest, EstimateLiveDataSize2) { ASSERT_EQ(4U, vstorage_.EstimateLiveDataSize()); } +TEST_F(VersionStorageInfoTest, SingleLevelBottommostData) { + // In case of a single level, the oldest L0 file is bottommost. This could be + // improved in case the L0 files cover disjoint key-ranges. + Add(0 /* level */, 1U /* file_number */, "A" /* smallest */, + "Z" /* largest */, 1U /* file_size */); + Add(0 /* level */, 2U /* file_number */, "A" /* smallest */, + "Z" /* largest */, 1U /* file_size */); + Add(0 /* level */, 3U /* file_number */, "0" /* smallest */, + "9" /* largest */, 1U /* file_size */); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(1, vstorage_.BottommostFiles().size()); + ASSERT_EQ(0, vstorage_.BottommostFiles()[0].first); + ASSERT_EQ(3U, vstorage_.BottommostFiles()[0].second->fd.GetNumber()); +} + +TEST_F(VersionStorageInfoTest, MultiLevelBottommostData) { + // In case of multiple levels, the oldest file for a key-range from each L1+ + // level is bottommost. This could be improved in case an L0 file contains the + // oldest data for some range of keys. + Add(0 /* level */, 1U /* file_number */, "A" /* smallest */, + "Z" /* largest */, 1U /* file_size */); + Add(0 /* level */, 2U /* file_number */, "0" /* smallest */, + "9" /* largest */, 1U /* file_size */); + Add(1 /* level */, 3U /* file_number */, "A" /* smallest */, + "D" /* largest */, 1U /* file_size */); + Add(2 /* level */, 4U /* file_number */, "E" /* smallest */, + "H" /* largest */, 1U /* file_size */); + Add(2 /* level */, 5U /* file_number */, "I" /* smallest */, + "L" /* largest */, 1U /* file_size */); + + UpdateVersionStorageInfo(); + + autovector> bottommost_files = + vstorage_.BottommostFiles(); + std::sort(bottommost_files.begin(), bottommost_files.end(), + [](const std::pair& lhs, + const std::pair& rhs) { + assert(lhs.second); + assert(rhs.second); + return lhs.second->fd.GetNumber() < rhs.second->fd.GetNumber(); + }); + ASSERT_EQ(3, bottommost_files.size()); + ASSERT_EQ(3U, bottommost_files[0].second->fd.GetNumber()); + ASSERT_EQ(4U, bottommost_files[1].second->fd.GetNumber()); + ASSERT_EQ(5U, bottommost_files[2].second->fd.GetNumber()); +} + TEST_F(VersionStorageInfoTest, GetOverlappingInputs) { // Two files that overlap at the range deletion tombstone sentinel. Add(1, 1U, {"a", 0, kTypeValue}, @@ -582,7 +635,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCEmpty) { constexpr double age_cutoff = 0.5; constexpr double force_threshold = 0.75; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } @@ -666,7 +720,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCSingleBatch) { { constexpr double age_cutoff = 0.1; constexpr double force_threshold = 0.0; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } @@ -677,7 +732,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCSingleBatch) { { constexpr double age_cutoff = 0.5; constexpr double force_threshold = 0.0; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } @@ -688,7 +744,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCSingleBatch) { { constexpr double age_cutoff = 1.0; constexpr double force_threshold = 0.6; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } @@ -699,7 +756,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCSingleBatch) { { constexpr double age_cutoff = 1.0; constexpr double force_threshold = 0.5; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); auto ssts_to_be_compacted = vstorage_.FilesMarkedForForcedBlobGC(); ASSERT_EQ(ssts_to_be_compacted.size(), 1); @@ -813,7 +871,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCMultipleBatches) { { constexpr double age_cutoff = 0.1; constexpr double force_threshold = 0.0; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } @@ -824,7 +883,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCMultipleBatches) { { constexpr double age_cutoff = 0.25; constexpr double force_threshold = 0.0; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } @@ -835,7 +895,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCMultipleBatches) { { constexpr double age_cutoff = 0.5; constexpr double force_threshold = 0.6; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } @@ -846,7 +907,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCMultipleBatches) { { constexpr double age_cutoff = 0.5; constexpr double force_threshold = 0.5; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); auto ssts_to_be_compacted = vstorage_.FilesMarkedForForcedBlobGC(); ASSERT_EQ(ssts_to_be_compacted.size(), 2); @@ -875,7 +937,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCMultipleBatches) { { constexpr double age_cutoff = 0.75; constexpr double force_threshold = 0.6; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } @@ -886,7 +949,8 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCMultipleBatches) { { constexpr double age_cutoff = 0.75; constexpr double force_threshold = 0.5; - vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true); auto ssts_to_be_compacted = vstorage_.FilesMarkedForForcedBlobGC(); ASSERT_EQ(ssts_to_be_compacted.size(), 2); @@ -913,7 +977,7 @@ class VersionStorageInfoTimestampTest : public VersionStorageInfoTestBase { VersionStorageInfoTimestampTest() : VersionStorageInfoTestBase(test::BytewiseComparatorWithU64TsWrapper()) { } - ~VersionStorageInfoTimestampTest() override {} + ~VersionStorageInfoTimestampTest() override = default; std::string Timestamp(uint64_t ts) const { std::string ret; PutFixed64(&ret, ts); @@ -967,7 +1031,7 @@ class FindLevelFileTest : public testing::Test { FindLevelFileTest() : disjoint_sorted_files_(true) {} - ~FindLevelFileTest() override {} + ~FindLevelFileTest() override = default; void LevelFileInit(size_t num = 0) { char* mem = arena_.AllocateAligned(num * sizeof(FdWithKeyRange)); @@ -1186,11 +1250,12 @@ class VersionSetTestBase { immutable_options_.fs = fs_; immutable_options_.clock = env_->GetSystemClock().get(); - versions_.reset( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + versions_.reset(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); reactive_versions_ = std::make_shared( dbname_, &db_options_, env_options_, table_cache_.get(), &write_buffer_manager_, &write_controller_, nullptr); @@ -1221,7 +1286,7 @@ class VersionSetTestBase { tmp_db_options.env = env_; std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); std::string db_id; - impl->GetDbIdentityFromIdentityFile(&db_id); + ASSERT_OK(impl->GetDbIdentityFromIdentityFile(&db_id)); new_db.SetDBId(db_id); } new_db.SetLogNumber(0); @@ -1277,7 +1342,7 @@ class VersionSetTestBase { void NewDB() { SequenceNumber last_seqno; std::unique_ptr log_writer; - SetIdentityFile(env_, dbname_); + ASSERT_OK(SetIdentityFile(env_, dbname_)); PrepareManifest(&column_families_, &last_seqno, &log_writer); log_writer.reset(); // Make "CURRENT" file point to the new manifest file. @@ -1290,11 +1355,12 @@ class VersionSetTestBase { } void ReopenDB() { - versions_.reset( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + versions_.reset(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); EXPECT_OK(versions_->Recover(column_families_, false)); } @@ -1349,6 +1415,8 @@ class VersionSetTestBase { new_cf.SetColumnFamily(new_id); new_cf.SetLogNumber(0); new_cf.SetComparatorName(cf_options.comparator->Name()); + new_cf.SetPersistUserDefinedTimestamps( + cf_options.persist_user_defined_timestamps); Status s; mutex_.Lock(); s = versions_->LogAndApply(/*column_family_data=*/nullptr, @@ -1800,11 +1868,12 @@ TEST_F(VersionSetTest, WalAddition) { // Recover a new VersionSet. { - std::unique_ptr new_versions( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + std::unique_ptr new_versions(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); ASSERT_OK(new_versions->Recover(column_families_, /*read_only=*/false)); const auto& wals = new_versions->GetWalSet().GetWals(); ASSERT_EQ(wals.size(), 1); @@ -1867,11 +1936,12 @@ TEST_F(VersionSetTest, WalCloseWithoutSync) { // Recover a new VersionSet. { - std::unique_ptr new_versions( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + std::unique_ptr new_versions(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); ASSERT_OK(new_versions->Recover(column_families_, false)); const auto& wals = new_versions->GetWalSet().GetWals(); ASSERT_EQ(wals.size(), 2); @@ -1920,11 +1990,12 @@ TEST_F(VersionSetTest, WalDeletion) { // Recover a new VersionSet, only the non-closed WAL should show up. { - std::unique_ptr new_versions( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + std::unique_ptr new_versions(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); ASSERT_OK(new_versions->Recover(column_families_, false)); const auto& wals = new_versions->GetWalSet().GetWals(); ASSERT_EQ(wals.size(), 1); @@ -1958,11 +2029,12 @@ TEST_F(VersionSetTest, WalDeletion) { // Recover from the new MANIFEST, only the non-closed WAL should show up. { - std::unique_ptr new_versions( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + std::unique_ptr new_versions(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); ASSERT_OK(new_versions->Recover(column_families_, false)); const auto& wals = new_versions->GetWalSet().GetWals(); ASSERT_EQ(wals.size(), 1); @@ -2078,11 +2150,12 @@ TEST_F(VersionSetTest, DeleteWalsBeforeNonExistingWalNumber) { // Recover a new VersionSet, WAL0 is deleted, WAL1 is not. { - std::unique_ptr new_versions( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + std::unique_ptr new_versions(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); ASSERT_OK(new_versions->Recover(column_families_, false)); const auto& wals = new_versions->GetWalSet().GetWals(); ASSERT_EQ(wals.size(), 1); @@ -2114,11 +2187,12 @@ TEST_F(VersionSetTest, DeleteAllWals) { // Recover a new VersionSet, all WALs are deleted. { - std::unique_ptr new_versions( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + std::unique_ptr new_versions(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); ASSERT_OK(new_versions->Recover(column_families_, false)); const auto& wals = new_versions->GetWalSet().GetWals(); ASSERT_EQ(wals.size(), 0); @@ -2156,11 +2230,12 @@ TEST_F(VersionSetTest, AtomicGroupWithWalEdits) { // Recover a new VersionSet, the min log number and the last WAL should be // kept. { - std::unique_ptr new_versions( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + std::unique_ptr new_versions(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); std::string db_id; ASSERT_OK( new_versions->Recover(column_families_, /*read_only=*/false, &db_id)); @@ -2175,6 +2250,99 @@ TEST_F(VersionSetTest, AtomicGroupWithWalEdits) { } } +TEST_F(VersionSetTest, OffpeakTimeInfoTest) { + Random rnd(test::RandomSeed()); + + // Sets off-peak time from 11:30PM to 4:30AM next day. + // Starting at 1:30PM, use mock sleep to make time pass + // and see if IsNowOffpeak() returns correctly per time changes + int now_hour = 13; + int now_minute = 30; + versions_->ChangeOffpeakTimeOption("23:30-04:30"); + + auto mock_clock = std::make_shared(env_->GetSystemClock()); + // Add some extra random days to current time + int days = rnd.Uniform(100); + mock_clock->SetCurrentTime(days * 86400 + now_hour * 3600 + now_minute * 60); + int64_t now; + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + + // Starting at 1:30PM. It's not off-peak + ASSERT_FALSE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Now it's at 4:30PM. Still not off-peak + mock_clock->MockSleepForSeconds(3 * 3600); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_FALSE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Now it's at 11:30PM. It's off-peak + mock_clock->MockSleepForSeconds(7 * 3600); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Now it's at 2:30AM next day. It's still off-peak + mock_clock->MockSleepForSeconds(3 * 3600); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Now it's at 4:30AM. It's still off-peak + mock_clock->MockSleepForSeconds(2 * 3600); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Sleep for one more minute. It's at 4:31AM It's no longer off-peak + mock_clock->MockSleepForSeconds(60); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_FALSE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Entire day offpeak + versions_->ChangeOffpeakTimeOption("00:00-23:59"); + // It doesn't matter what time it is. It should be just offpeak. + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Mock Sleep for 3 hours. It's still off-peak + mock_clock->MockSleepForSeconds(3 * 3600); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Mock Sleep for 20 hours. It's still off-peak + mock_clock->MockSleepForSeconds(20 * 3600); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Mock Sleep for 59 minutes. It's still off-peak + mock_clock->MockSleepForSeconds(59 * 60); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Mock Sleep for 59 seconds. It's still off-peak + mock_clock->MockSleepForSeconds(59); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + + // Mock Sleep for 1 second (exactly 24h passed). It's still off-peak + mock_clock->MockSleepForSeconds(1); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); + // Another second for sanity check + mock_clock->MockSleepForSeconds(1); + ASSERT_OK(mock_clock.get()->GetCurrentTime(&now)); + ASSERT_TRUE( + versions_->offpeak_time_option().GetOffpeakTimeInfo(now).is_now_offpeak); +} + TEST_F(VersionStorageInfoTest, AddRangeDeletionCompensatedFileSize) { // Tests that compensated range deletion size is added to compensated file // size. @@ -2221,11 +2389,12 @@ class VersionSetWithTimestampTest : public VersionSetTest { } void VerifyFullHistoryTsLow(uint64_t expected_ts_low) { - std::unique_ptr vset( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + std::unique_ptr vset(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); ASSERT_OK(vset->Recover(column_families_, /*read_only=*/false, /*db_id=*/nullptr)); for (auto* cfd : *(vset->GetColumnFamilySet())) { @@ -2860,11 +3029,12 @@ class VersionSetTestEmptyDb assert(nullptr != log_writer); VersionEdit new_db; if (db_options_.write_dbid_to_manifest) { + ASSERT_OK(SetIdentityFile(env_, dbname_)); DBOptions tmp_db_options; tmp_db_options.env = env_; std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); std::string db_id; - impl->GetDbIdentityFromIdentityFile(&db_id); + ASSERT_OK(impl->GetDbIdentityFromIdentityFile(&db_id)); new_db.SetDBId(db_id); } const std::string manifest_path = DescriptorFileName(dbname_, 1); @@ -3194,7 +3364,7 @@ class VersionSetTestMissingFiles : public VersionSetTestBase, tmp_db_options.env = env_; std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); std::string db_id; - impl->GetDbIdentityFromIdentityFile(&db_id); + ASSERT_OK(impl->GetDbIdentityFromIdentityFile(&db_id)); new_db.SetDBId(db_id); } { @@ -3541,7 +3711,7 @@ class ChargeFileMetadataTestWithParam : public ChargeFileMetadataTest, public testing::WithParamInterface { public: - ChargeFileMetadataTestWithParam() {} + ChargeFileMetadataTestWithParam() = default; }; INSTANTIATE_TEST_CASE_P( diff --git a/db/version_util.h b/db/version_util.h index e39f25571..acb27749b 100644 --- a/db/version_util.h +++ b/db/version_util.h @@ -25,7 +25,9 @@ class OfflineManifestWriter { options.table_cache_numshardbits)), versions_(db_path, &immutable_db_options_, sopt_, tc_.get(), &wb_, &wc_, /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "") {} + /*db_id=*/"", /*db_session_id=*/"", + options.daily_offpeak_time_utc, + /*error_handler=*/nullptr) {} Status Recover(const std::vector& column_families) { return versions_.Recover(column_families, /*read_only*/ false, diff --git a/db/wal_manager.cc b/db/wal_manager.cc index 400b2e58b..2b384e7d2 100644 --- a/db/wal_manager.cc +++ b/db/wal_manager.cc @@ -153,10 +153,11 @@ void WalManager::PurgeObsoleteWALFiles() { return; } uint64_t const now_seconds = static_cast(current_time); - uint64_t const time_to_check = (ttl_enabled && !size_limit_enabled) - ? db_options_.WAL_ttl_seconds / 2 - : kDefaultIntervalToDeleteObsoleteWAL; - + uint64_t const time_to_check = + ttl_enabled + ? std::min(kDefaultIntervalToDeleteObsoleteWAL, + std::max(uint64_t{1}, db_options_.WAL_ttl_seconds / 2)) + : kDefaultIntervalToDeleteObsoleteWAL; if (purge_wal_files_last_run_ + time_to_check > now_seconds) { return; } diff --git a/db/wal_manager_test.cc b/db/wal_manager_test.cc index 0144e1846..0ead57ae8 100644 --- a/db/wal_manager_test.cc +++ b/db/wal_manager_test.cc @@ -50,11 +50,12 @@ class WalManagerTest : public testing::Test { db_options_.fs = env_->GetFileSystem(); db_options_.clock = env_->GetSystemClock().get(); - versions_.reset( - new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), - &write_buffer_manager_, &write_controller_, - /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ "")); + versions_.reset(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"", + /*error_handler=*/nullptr)); wal_manager_.reset( new WalManager(db_options_, env_options_, nullptr /*IOTracer*/)); @@ -333,4 +334,3 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - diff --git a/db/wide/db_wide_basic_test.cc b/db/wide/db_wide_basic_test.cc index 536a543e6..15d2fdff7 100644 --- a/db/wide/db_wide_basic_test.cc +++ b/db/wide/db_wide_basic_test.cc @@ -10,6 +10,7 @@ #include "db/db_test_util.h" #include "port/stack_trace.h" #include "test_util/testutil.h" +#include "util/overload.h" #include "utilities/merge_operators.h" namespace ROCKSDB_NAMESPACE { @@ -93,7 +94,7 @@ TEST_F(DBWideBasicTest, PutEntity) { std::array statuses; db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, - &keys[0], &values[0], &statuses[0]); + keys.data(), values.data(), statuses.data()); ASSERT_OK(statuses[0]); ASSERT_EQ(values[0], first_value_of_default_column); @@ -113,7 +114,7 @@ TEST_F(DBWideBasicTest, PutEntity) { std::array statuses; db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys, - &keys[0], &results[0], &statuses[0]); + keys.data(), results.data(), statuses.data()); ASSERT_OK(statuses[0]); ASSERT_EQ(results[0].columns(), first_columns); @@ -208,6 +209,11 @@ TEST_F(DBWideBasicTest, PutEntity) { ASSERT_OK(Flush()); verify(); + + // Reopen as Readonly DB and verify + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + verify(); } TEST_F(DBWideBasicTest, PutEntityColumnFamily) { @@ -230,6 +236,144 @@ TEST_F(DBWideBasicTest, PutEntityColumnFamily) { ASSERT_OK(db_->Write(WriteOptions(), &batch)); } +TEST_F(DBWideBasicTest, GetEntityAsPinnableAttributeGroups) { + Options options = GetDefaultOptions(); + CreateAndReopenWithCF({"hot_cf", "cold_cf"}, options); + + constexpr int kDefaultCfHandleIndex = 0; + constexpr int kHotCfHandleIndex = 1; + constexpr int kColdCfHandleIndex = 2; + + constexpr char first_key[] = "first"; + WideColumns first_default_columns{ + {"default_cf_col_1_name", "first_key_default_cf_col_1_value"}, + {"default_cf_col_2_name", "first_key_default_cf_col_2_value"}}; + WideColumns first_hot_columns{ + {"hot_cf_col_1_name", "first_key_hot_cf_col_1_value"}, + {"hot_cf_col_2_name", "first_key_hot_cf_col_2_value"}}; + WideColumns first_cold_columns{ + {"cold_cf_col_1_name", "first_key_cold_cf_col_1_value"}}; + + constexpr char second_key[] = "second"; + WideColumns second_hot_columns{ + {"hot_cf_col_1_name", "second_key_hot_cf_col_1_value"}}; + WideColumns second_cold_columns{ + {"cold_cf_col_1_name", "second_key_cold_cf_col_1_value"}}; + + AttributeGroups first_key_attribute_groups{ + AttributeGroup(handles_[kDefaultCfHandleIndex], first_default_columns), + AttributeGroup(handles_[kHotCfHandleIndex], first_hot_columns), + AttributeGroup(handles_[kColdCfHandleIndex], first_cold_columns)}; + AttributeGroups second_key_attribute_groups{ + AttributeGroup(handles_[kHotCfHandleIndex], second_hot_columns), + AttributeGroup(handles_[kColdCfHandleIndex], second_cold_columns)}; + ASSERT_OK( + db_->PutEntity(WriteOptions(), first_key, first_key_attribute_groups)); + ASSERT_OK( + db_->PutEntity(WriteOptions(), second_key, second_key_attribute_groups)); + + std::vector all_cfs = handles_; + std::vector default_and_hot_cfs{ + {handles_[kDefaultCfHandleIndex], handles_[kHotCfHandleIndex]}}; + std::vector hot_and_cold_cfs{ + {handles_[kHotCfHandleIndex], handles_[kColdCfHandleIndex]}}; + std::vector default_null_and_hot_cfs{ + handles_[kDefaultCfHandleIndex], nullptr, handles_[kHotCfHandleIndex], + nullptr}; + auto create_result = + [](const std::vector& column_families) + -> PinnableAttributeGroups { + PinnableAttributeGroups result; + for (size_t i = 0; i < column_families.size(); ++i) { + result.emplace_back(column_families[i]); + } + return result; + }; + { + // Case 1. Invalid Argument (passing in null CF) + AttributeGroups ag{ + AttributeGroup(nullptr, first_default_columns), + AttributeGroup(handles_[kHotCfHandleIndex], first_hot_columns)}; + ASSERT_NOK(db_->PutEntity(WriteOptions(), first_key, ag)); + + PinnableAttributeGroups result = create_result(default_null_and_hot_cfs); + Status s = db_->GetEntity(ReadOptions(), first_key, &result); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + // Valid CF, but failed with Incomplete status due to other attribute groups + ASSERT_TRUE(result[0].status().IsIncomplete()); + // Null CF + ASSERT_TRUE(result[1].status().IsInvalidArgument()); + // Valid CF, but failed with Incomplete status due to other attribute groups + ASSERT_TRUE(result[2].status().IsIncomplete()); + // Null CF, but failed with Incomplete status because the nullcheck break + // out early in the loop + ASSERT_TRUE(result[3].status().IsIncomplete()); + } + { + // Case 2. Get first key from default cf and hot_cf and second key from + // hot_cf and cold_cf + constexpr size_t num_column_families = 2; + PinnableAttributeGroups first_key_result = + create_result(default_and_hot_cfs); + PinnableAttributeGroups second_key_result = create_result(hot_and_cold_cfs); + + // GetEntity for first_key + ASSERT_OK(db_->GetEntity(ReadOptions(), first_key, &first_key_result)); + ASSERT_EQ(num_column_families, first_key_result.size()); + // We expect to get values for all keys and CFs + for (size_t i = 0; i < num_column_families; ++i) { + ASSERT_OK(first_key_result[i].status()); + } + // verify values for first key (default cf and hot cf) + ASSERT_EQ(first_default_columns, first_key_result[0].columns()); + ASSERT_EQ(first_hot_columns, first_key_result[1].columns()); + + // GetEntity for second_key + ASSERT_OK(db_->GetEntity(ReadOptions(), second_key, &second_key_result)); + ASSERT_EQ(num_column_families, second_key_result.size()); + // We expect to get values for all keys and CFs + for (size_t i = 0; i < num_column_families; ++i) { + ASSERT_OK(second_key_result[i].status()); + } + // verify values for second key (hot cf and cold cf) + ASSERT_EQ(second_hot_columns, second_key_result[0].columns()); + ASSERT_EQ(second_cold_columns, second_key_result[1].columns()); + } + { + // Case 3. Get first key and second key from all cfs. For the second key, we + // don't expect to get columns from default cf. + constexpr size_t num_column_families = 3; + PinnableAttributeGroups first_key_result = create_result(all_cfs); + PinnableAttributeGroups second_key_result = create_result(all_cfs); + + // GetEntity for first_key + ASSERT_OK(db_->GetEntity(ReadOptions(), first_key, &first_key_result)); + ASSERT_EQ(num_column_families, first_key_result.size()); + // We expect to get values for all keys and CFs + for (size_t i = 0; i < num_column_families; ++i) { + ASSERT_OK(first_key_result[i].status()); + } + // verify values for first key + ASSERT_EQ(first_default_columns, first_key_result[0].columns()); + ASSERT_EQ(first_hot_columns, first_key_result[1].columns()); + ASSERT_EQ(first_cold_columns, first_key_result[2].columns()); + + // GetEntity for second_key + ASSERT_OK(db_->GetEntity(ReadOptions(), second_key, &second_key_result)); + ASSERT_EQ(num_column_families, second_key_result.size()); + // key does not exist in default cf + ASSERT_NOK(second_key_result[0].status()); + ASSERT_TRUE(second_key_result[0].status().IsNotFound()); + + // verify values for second key (hot cf and cold cf) + ASSERT_OK(second_key_result[1].status()); + ASSERT_OK(second_key_result[2].status()); + ASSERT_EQ(second_hot_columns, second_key_result[1].columns()); + ASSERT_EQ(second_cold_columns, second_key_result[2].columns()); + } +} + TEST_F(DBWideBasicTest, MultiCFMultiGetEntity) { Options options = GetDefaultOptions(); CreateAndReopenWithCF({"corinthian"}, options); @@ -254,8 +398,8 @@ TEST_F(DBWideBasicTest, MultiCFMultiGetEntity) { std::array results; std::array statuses; - db_->MultiGetEntity(ReadOptions(), num_keys, &column_families[0], &keys[0], - &results[0], &statuses[0]); + db_->MultiGetEntity(ReadOptions(), num_keys, column_families.data(), + keys.data(), results.data(), statuses.data()); ASSERT_OK(statuses[0]); ASSERT_EQ(results[0].columns(), first_columns); @@ -264,6 +408,162 @@ TEST_F(DBWideBasicTest, MultiCFMultiGetEntity) { ASSERT_EQ(results[1].columns(), second_columns); } +TEST_F(DBWideBasicTest, MultiCFMultiGetEntityAsPinnableAttributeGroups) { + Options options = GetDefaultOptions(); + CreateAndReopenWithCF({"hot_cf", "cold_cf"}, options); + + constexpr int kDefaultCfHandleIndex = 0; + constexpr int kHotCfHandleIndex = 1; + constexpr int kColdCfHandleIndex = 2; + + constexpr char first_key[] = "first"; + WideColumns first_default_columns{ + {"default_cf_col_1_name", "first_key_default_cf_col_1_value"}, + {"default_cf_col_2_name", "first_key_default_cf_col_2_value"}}; + WideColumns first_hot_columns{ + {"hot_cf_col_1_name", "first_key_hot_cf_col_1_value"}, + {"hot_cf_col_2_name", "first_key_hot_cf_col_2_value"}}; + WideColumns first_cold_columns{ + {"cold_cf_col_1_name", "first_key_cold_cf_col_1_value"}}; + constexpr char second_key[] = "second"; + WideColumns second_hot_columns{ + {"hot_cf_col_1_name", "second_key_hot_cf_col_1_value"}}; + WideColumns second_cold_columns{ + {"cold_cf_col_1_name", "second_key_cold_cf_col_1_value"}}; + + AttributeGroups first_key_attribute_groups{ + AttributeGroup(handles_[kDefaultCfHandleIndex], first_default_columns), + AttributeGroup(handles_[kHotCfHandleIndex], first_hot_columns), + AttributeGroup(handles_[kColdCfHandleIndex], first_cold_columns)}; + AttributeGroups second_key_attribute_groups{ + AttributeGroup(handles_[kHotCfHandleIndex], second_hot_columns), + AttributeGroup(handles_[kColdCfHandleIndex], second_cold_columns)}; + + ASSERT_OK( + db_->PutEntity(WriteOptions(), first_key, first_key_attribute_groups)); + ASSERT_OK( + db_->PutEntity(WriteOptions(), second_key, second_key_attribute_groups)); + + constexpr size_t num_keys = 2; + std::array keys = {first_key, second_key}; + std::vector all_cfs = handles_; + std::vector default_and_hot_cfs{ + {handles_[kDefaultCfHandleIndex], handles_[kHotCfHandleIndex]}}; + std::vector hot_and_cold_cfs{ + {handles_[kHotCfHandleIndex], handles_[kColdCfHandleIndex]}}; + std::vector null_and_hot_cfs{ + nullptr, handles_[kHotCfHandleIndex], nullptr}; + auto create_result = + [](const std::vector& column_families) + -> PinnableAttributeGroups { + PinnableAttributeGroups result; + for (size_t i = 0; i < column_families.size(); ++i) { + result.emplace_back(column_families[i]); + } + return result; + }; + { + // Check for invalid read option argument + ReadOptions read_options; + read_options.io_activity = Env::IOActivity::kGetEntity; + std::vector results; + for (size_t i = 0; i < num_keys; ++i) { + results.emplace_back(create_result(all_cfs)); + } + db_->MultiGetEntity(read_options, num_keys, keys.data(), results.data()); + for (size_t i = 0; i < num_keys; ++i) { + for (size_t j = 0; j < all_cfs.size(); ++j) { + ASSERT_NOK(results[i][j].status()); + ASSERT_TRUE(results[i][j].status().IsInvalidArgument()); + } + } + // Check for invalid column family in Attribute Group result + results.clear(); + results.emplace_back(create_result(null_and_hot_cfs)); + results.emplace_back(create_result(all_cfs)); + db_->MultiGetEntity(ReadOptions(), num_keys, keys.data(), results.data()); + + // First one failed due to null CFs in the AttributeGroup + // Null CF + ASSERT_NOK(results[0][0].status()); + ASSERT_TRUE(results[0][0].status().IsInvalidArgument()); + // Valid CF, but failed with incomplete status because of other attribute + // groups + ASSERT_NOK(results[0][1].status()); + ASSERT_TRUE(results[0][1].status().IsIncomplete()); + // Null CF + ASSERT_NOK(results[0][2].status()); + ASSERT_TRUE(results[0][2].status().IsInvalidArgument()); + + // Second one failed with Incomplete because first one failed + ASSERT_NOK(results[1][0].status()); + ASSERT_TRUE(results[1][0].status().IsIncomplete()); + ASSERT_NOK(results[1][1].status()); + ASSERT_TRUE(results[1][1].status().IsIncomplete()); + ASSERT_NOK(results[1][2].status()); + ASSERT_TRUE(results[1][2].status().IsIncomplete()); + } + { + // Case 1. Get first key from default cf and hot_cf and second key from + // hot_cf and cold_cf + std::vector results; + PinnableAttributeGroups first_key_result = + create_result(default_and_hot_cfs); + PinnableAttributeGroups second_key_result = create_result(hot_and_cold_cfs); + results.emplace_back(std::move(first_key_result)); + results.emplace_back(std::move(second_key_result)); + + db_->MultiGetEntity(ReadOptions(), num_keys, keys.data(), results.data()); + ASSERT_EQ(2, results.size()); + // We expect to get values for all keys and CFs + for (size_t i = 0; i < num_keys; ++i) { + for (size_t j = 0; j < 2; ++j) { + ASSERT_OK(results[i][j].status()); + } + } + // verify values for first key (default cf and hot cf) + ASSERT_EQ(2, results[0].size()); + ASSERT_EQ(first_default_columns, results[0][0].columns()); + ASSERT_EQ(first_hot_columns, results[0][1].columns()); + + // verify values for second key (hot cf and cold cf) + ASSERT_EQ(2, results[1].size()); + ASSERT_EQ(second_hot_columns, results[1][0].columns()); + ASSERT_EQ(second_cold_columns, results[1][1].columns()); + } + { + // Case 2. Get first key and second key from all cfs. For the second key, we + // don't expect to get columns from default cf. + std::vector results; + PinnableAttributeGroups first_key_result = create_result(all_cfs); + PinnableAttributeGroups second_key_result = create_result(all_cfs); + results.emplace_back(std::move(first_key_result)); + results.emplace_back(std::move(second_key_result)); + + db_->MultiGetEntity(ReadOptions(), num_keys, keys.data(), results.data()); + // verify first key + for (size_t i = 0; i < all_cfs.size(); ++i) { + ASSERT_OK(results[0][i].status()); + } + ASSERT_EQ(3, results[0].size()); + ASSERT_EQ(first_default_columns, results[0][0].columns()); + ASSERT_EQ(first_hot_columns, results[0][1].columns()); + ASSERT_EQ(first_cold_columns, results[0][2].columns()); + + // verify second key + // key does not exist in default cf + ASSERT_NOK(results[1][0].status()); + ASSERT_TRUE(results[1][0].status().IsNotFound()); + ASSERT_TRUE(results[1][0].columns().empty()); + + // key exists in hot_cf and cold_cf + ASSERT_OK(results[1][1].status()); + ASSERT_EQ(second_hot_columns, results[1][1].columns()); + ASSERT_OK(results[1][2].status()); + ASSERT_EQ(second_cold_columns, results[1][2].columns()); + } +} + TEST_F(DBWideBasicTest, MergePlainKeyValue) { Options options = GetDefaultOptions(); options.create_if_missing = true; @@ -342,7 +642,7 @@ TEST_F(DBWideBasicTest, MergePlainKeyValue) { std::array statuses; db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys, - &keys[0], &results[0], &statuses[0]); + keys.data(), results.data(), statuses.data()); ASSERT_OK(statuses[0]); ASSERT_EQ(results[0].columns(), expected_first_columns); @@ -522,7 +822,7 @@ TEST_F(DBWideBasicTest, MergeEntity) { std::array statuses; db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, - &keys[0], &values[0], &statuses[0]); + keys.data(), values.data(), statuses.data()); ASSERT_EQ(values[0], first_expected_default); ASSERT_OK(statuses[0]); @@ -539,7 +839,7 @@ TEST_F(DBWideBasicTest, MergeEntity) { std::array statuses; db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys, - &keys[0], &results[0], &statuses[0]); + keys.data(), results.data(), statuses.data()); ASSERT_OK(statuses[0]); ASSERT_EQ(results[0].columns(), first_expected_columns); @@ -600,7 +900,7 @@ TEST_F(DBWideBasicTest, MergeEntity) { int number_of_operands = 0; ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), - first_key, &merge_operands[0], + first_key, merge_operands.data(), &get_merge_opts, &number_of_operands)); ASSERT_EQ(number_of_operands, num_merge_operands); @@ -613,7 +913,7 @@ TEST_F(DBWideBasicTest, MergeEntity) { int number_of_operands = 0; ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), - second_key, &merge_operands[0], + second_key, merge_operands.data(), &get_merge_opts, &number_of_operands)); ASSERT_EQ(number_of_operands, num_merge_operands); @@ -633,7 +933,7 @@ TEST_F(DBWideBasicTest, MergeEntity) { int number_of_operands = 0; ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), - first_key, &merge_operands[0], + first_key, merge_operands.data(), &get_merge_opts, &number_of_operands)); ASSERT_EQ(number_of_operands, num_merge_operands); @@ -645,7 +945,7 @@ TEST_F(DBWideBasicTest, MergeEntity) { int number_of_operands = 0; ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), - second_key, &merge_operands[0], + second_key, merge_operands.data(), &get_merge_opts, &number_of_operands)); ASSERT_EQ(number_of_operands, num_merge_operands); @@ -685,6 +985,397 @@ TEST_F(DBWideBasicTest, MergeEntity) { verify_merge_ops_post_compaction(); } +class DBWideMergeV3Test : public DBWideBasicTest { + protected: + void RunTest(const WideColumns& first_expected, + const WideColumns& second_expected, + const WideColumns& third_expected) { + // Note: we'll take some snapshots to prevent merging during flush + snapshots_.reserve(6); + + // Test reading from memtables + WriteKeyValues(); + VerifyKeyValues(first_expected, second_expected, third_expected); + VerifyMergeOperandCount(first_key, 2); + VerifyMergeOperandCount(second_key, 3); + VerifyMergeOperandCount(third_key, 3); + + // Test reading from SST files + ASSERT_OK(Flush()); + VerifyKeyValues(first_expected, second_expected, third_expected); + VerifyMergeOperandCount(first_key, 2); + VerifyMergeOperandCount(second_key, 3); + VerifyMergeOperandCount(third_key, 3); + + // Test reading from SSTs after compaction. Note that we write the same KVs + // and flush again so we have two overlapping files. We also release the + // snapshots so that the compaction can merge all keys. + WriteKeyValues(); + ASSERT_OK(Flush()); + + snapshots_.clear(); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /* begin */ nullptr, + /* end */ nullptr)); + VerifyKeyValues(first_expected, second_expected, third_expected); + VerifyMergeOperandCount(first_key, 1); + VerifyMergeOperandCount(second_key, 1); + VerifyMergeOperandCount(third_key, 1); + } + + void WriteKeyValues() { + // Base values + ASSERT_OK(db_->Delete(WriteOptions(), db_->DefaultColumnFamily(), + first_key)); // no base value + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), second_key, + second_base_value)); // plain base value + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), + third_key, + third_columns)); // wide-column base value + + snapshots_.emplace_back(db_); + + // First round of merge operands + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), first_key, + first_merge_op1)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), second_key, + second_merge_op1)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), third_key, + third_merge_op1)); + + snapshots_.emplace_back(db_); + + // Second round of merge operands + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), first_key, + first_merge_op2)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), second_key, + second_merge_op2)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), third_key, + third_merge_op2)); + + snapshots_.emplace_back(db_); + } + + void VerifyKeyValues(const WideColumns& first_expected, + const WideColumns& second_expected, + const WideColumns& third_expected) { + assert(!first_expected.empty() && + first_expected[0].name() == kDefaultWideColumnName); + assert(!second_expected.empty() && + second_expected[0].name() == kDefaultWideColumnName); + assert(!third_expected.empty() && + third_expected[0].name() == kDefaultWideColumnName); + + // Get + { + PinnableSlice result; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), first_key, + &result)); + ASSERT_EQ(result, first_expected[0].value()); + } + + { + PinnableSlice result; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), second_key, + &result)); + ASSERT_EQ(result, second_expected[0].value()); + } + + { + PinnableSlice result; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), third_key, + &result)); + ASSERT_EQ(result, third_expected[0].value()); + } + + // MultiGet + { + std::array keys{{first_key, second_key, third_key}}; + std::array values; + std::array statuses; + + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + keys.data(), values.data(), statuses.data()); + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_expected[0].value()); + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], second_expected[0].value()); + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], third_expected[0].value()); + } + + // GetEntity + { + PinnableWideColumns result; + + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &result)); + ASSERT_EQ(result.columns(), first_expected); + } + + { + PinnableWideColumns result; + + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &result)); + ASSERT_EQ(result.columns(), second_expected); + } + + { + PinnableWideColumns result; + + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + third_key, &result)); + ASSERT_EQ(result.columns(), third_expected); + } + + // MultiGetEntity + { + std::array keys{{first_key, second_key, third_key}}; + std::array results; + std::array statuses; + + db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + keys.data(), results.data(), statuses.data()); + ASSERT_OK(statuses[0]); + ASSERT_EQ(results[0].columns(), first_expected); + ASSERT_OK(statuses[1]); + ASSERT_EQ(results[1].columns(), second_expected); + ASSERT_OK(statuses[2]); + ASSERT_EQ(results[2].columns(), third_expected); + } + + // Iterator + { + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_expected[0].value()); + ASSERT_EQ(iter->columns(), first_expected); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), second_expected[0].value()); + ASSERT_EQ(iter->columns(), second_expected); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), third_expected[0].value()); + ASSERT_EQ(iter->columns(), third_expected); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), third_expected[0].value()); + ASSERT_EQ(iter->columns(), third_expected); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), second_expected[0].value()); + ASSERT_EQ(iter->columns(), second_expected); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_expected[0].value()); + ASSERT_EQ(iter->columns(), first_expected); + + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } + } + + void VerifyMergeOperandCount(const Slice& key, int expected_merge_ops) { + GetMergeOperandsOptions get_merge_opts; + get_merge_opts.expected_max_number_of_operands = expected_merge_ops; + + std::vector merge_operands(expected_merge_ops); + int number_of_operands = 0; + + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + key, merge_operands.data(), &get_merge_opts, + &number_of_operands)); + ASSERT_EQ(number_of_operands, expected_merge_ops); + } + + std::vector snapshots_; + + static constexpr size_t num_keys = 3; + + static constexpr char first_key[] = "first"; + static constexpr char first_merge_op1[] = "hello"; + static constexpr char first_merge_op1_upper[] = "HELLO"; + static constexpr char first_merge_op2[] = "world"; + static constexpr char first_merge_op2_upper[] = "WORLD"; + + static constexpr char second_key[] = "second"; + static constexpr char second_base_value[] = "foo"; + static constexpr char second_base_value_upper[] = "FOO"; + static constexpr char second_merge_op1[] = "bar"; + static constexpr char second_merge_op1_upper[] = "BAR"; + static constexpr char second_merge_op2[] = "baz"; + static constexpr char second_merge_op2_upper[] = "BAZ"; + + static constexpr char third_key[] = "third"; + static const WideColumns third_columns; + static constexpr char third_merge_op1[] = "three"; + static constexpr char third_merge_op1_upper[] = "THREE"; + static constexpr char third_merge_op2[] = "four"; + static constexpr char third_merge_op2_upper[] = "FOUR"; +}; + +const WideColumns DBWideMergeV3Test::third_columns{{"one", "ONE"}, + {"two", "TWO"}}; + +TEST_F(DBWideMergeV3Test, MergeV3WideColumnOutput) { + // A test merge operator that always returns a wide-column result. It adds any + // base values and merge operands to a single wide-column entity, and converts + // all column values to uppercase. In addition, it puts "none", "plain", or + // "wide" into the value of the default column depending on the type of the + // base value (if any). + static constexpr char kNone[] = "none"; + static constexpr char kPlain[] = "plain"; + static constexpr char kWide[] = "wide"; + + class WideColumnOutputMergeOperator : public MergeOperator { + public: + bool FullMergeV3(const MergeOperationInputV3& merge_in, + MergeOperationOutputV3* merge_out) const override { + assert(merge_out); + + merge_out->new_value = MergeOperationOutputV3::NewColumns(); + auto& new_columns = + std::get(merge_out->new_value); + + auto upper = [](std::string str) { + for (char& c : str) { + c = static_cast(std::toupper(static_cast(c))); + } + + return str; + }; + + std::visit(overload{[&](const std::monostate&) { + new_columns.emplace_back( + kDefaultWideColumnName.ToString(), kNone); + }, + [&](const Slice& value) { + new_columns.emplace_back( + kDefaultWideColumnName.ToString(), kPlain); + + const std::string val = value.ToString(); + new_columns.emplace_back(val, upper(val)); + }, + [&](const WideColumns& columns) { + new_columns.emplace_back( + kDefaultWideColumnName.ToString(), kWide); + + for (const auto& column : columns) { + new_columns.emplace_back( + column.name().ToString(), + upper(column.value().ToString())); + } + }}, + merge_in.existing_value); + + for (const auto& operand : merge_in.operand_list) { + const std::string op = operand.ToString(); + new_columns.emplace_back(op, upper(op)); + } + + return true; + } + + const char* Name() const override { + return "WideColumnOutputMergeOperator"; + } + }; + + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.merge_operator = std::make_shared(); + Reopen(options); + + // Expected results + // Lexicographical order: [default] < hello < world + const WideColumns first_expected{{kDefaultWideColumnName, kNone}, + {first_merge_op1, first_merge_op1_upper}, + {first_merge_op2, first_merge_op2_upper}}; + // Lexicographical order: [default] < bar < baz < foo + const WideColumns second_expected{ + {kDefaultWideColumnName, kPlain}, + {second_merge_op1, second_merge_op1_upper}, + {second_merge_op2, second_merge_op2_upper}, + {second_base_value, second_base_value_upper}}; + // Lexicographical order: [default] < four < one < three < two + const WideColumns third_expected{ + {kDefaultWideColumnName, kWide}, + {third_merge_op2, third_merge_op2_upper}, + {third_columns[0].name(), third_columns[0].value()}, + {third_merge_op1, third_merge_op1_upper}, + {third_columns[1].name(), third_columns[1].value()}}; + + RunTest(first_expected, second_expected, third_expected); +} + +TEST_F(DBWideMergeV3Test, MergeV3PlainOutput) { + // A test merge operator that always returns a plain value as result, namely + // the total number of operands serialized as a string. Base values are also + // counted as operands; specifically, a plain base value is counted as one + // operand, while a wide-column base value is counted as as many operands as + // the number of columns. + class PlainOutputMergeOperator : public MergeOperator { + public: + bool FullMergeV3(const MergeOperationInputV3& merge_in, + MergeOperationOutputV3* merge_out) const override { + assert(merge_out); + + size_t count = 0; + std::visit( + overload{[&](const std::monostate&) {}, + [&](const Slice&) { count = 1; }, + [&](const WideColumns& columns) { count = columns.size(); }}, + merge_in.existing_value); + + count += merge_in.operand_list.size(); + + merge_out->new_value = std::string(); + std::get(merge_out->new_value) = std::to_string(count); + + return true; + } + + const char* Name() const override { return "PlainOutputMergeOperator"; } + }; + + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.merge_operator = std::make_shared(); + Reopen(options); + + const WideColumns first_expected{{kDefaultWideColumnName, "2"}}; + const WideColumns second_expected{{kDefaultWideColumnName, "3"}}; + const WideColumns third_expected{{kDefaultWideColumnName, "4"}}; + + RunTest(first_expected, second_expected, third_expected); +} + TEST_F(DBWideBasicTest, CompactionFilter) { Options options = GetDefaultOptions(); options.create_if_missing = true; diff --git a/db/wide/wide_column_serialization.cc b/db/wide/wide_column_serialization.cc index f62143c40..bb3f29584 100644 --- a/db/wide/wide_column_serialization.cc +++ b/db/wide/wide_column_serialization.cc @@ -9,17 +9,16 @@ #include #include +#include "db/wide/wide_columns_helper.h" #include "rocksdb/slice.h" #include "util/autovector.h" #include "util/coding.h" namespace ROCKSDB_NAMESPACE { -Status WideColumnSerialization::SerializeImpl(const Slice* value_of_default, - const WideColumns& columns, - std::string& output) { - const size_t num_columns = - value_of_default ? columns.size() + 1 : columns.size(); +Status WideColumnSerialization::Serialize(const WideColumns& columns, + std::string& output) { + const size_t num_columns = columns.size(); if (num_columns > static_cast(std::numeric_limits::max())) { return Status::InvalidArgument("Too many wide columns"); @@ -30,17 +29,6 @@ Status WideColumnSerialization::SerializeImpl(const Slice* value_of_default, PutVarint32(&output, static_cast(num_columns)); const Slice* prev_name = nullptr; - if (value_of_default) { - if (value_of_default->size() > - static_cast(std::numeric_limits::max())) { - return Status::InvalidArgument("Wide column value too long"); - } - - PutLengthPrefixedSlice(&output, kDefaultWideColumnName); - PutVarint32(&output, static_cast(value_of_default->size())); - - prev_name = &kDefaultWideColumnName; - } for (size_t i = 0; i < columns.size(); ++i) { const WideColumn& column = columns[i]; @@ -67,10 +55,6 @@ Status WideColumnSerialization::SerializeImpl(const Slice* value_of_default, prev_name = &name; } - if (value_of_default) { - output.append(value_of_default->data(), value_of_default->size()); - } - for (const auto& column : columns) { const Slice& value = column.value(); @@ -169,12 +153,12 @@ Status WideColumnSerialization::GetValueOfDefaultColumn(Slice& input, return s; } - if (columns.empty() || columns[0].name() != kDefaultWideColumnName) { + if (!WideColumnsHelper::HasDefaultColumn(columns)) { value.clear(); return Status::OK(); } - value = columns[0].value(); + value = WideColumnsHelper::GetDefaultColumn(columns); return Status::OK(); } diff --git a/db/wide/wide_column_serialization.h b/db/wide/wide_column_serialization.h index f0ffbd392..bb92db04f 100644 --- a/db/wide/wide_column_serialization.h +++ b/db/wide/wide_column_serialization.h @@ -44,9 +44,6 @@ class Slice; class WideColumnSerialization { public: static Status Serialize(const WideColumns& columns, std::string& output); - static Status Serialize(const Slice& value_of_default, - const WideColumns& other_columns, - std::string& output); static Status Deserialize(Slice& input, WideColumns& columns); @@ -55,23 +52,6 @@ class WideColumnSerialization { static Status GetValueOfDefaultColumn(Slice& input, Slice& value); static constexpr uint32_t kCurrentVersion = 1; - - private: - static Status SerializeImpl(const Slice* value_of_default, - const WideColumns& columns, std::string& output); }; -inline Status WideColumnSerialization::Serialize(const WideColumns& columns, - std::string& output) { - constexpr Slice* value_of_default = nullptr; - - return SerializeImpl(value_of_default, columns, output); -} - -inline Status WideColumnSerialization::Serialize( - const Slice& value_of_default, const WideColumns& other_columns, - std::string& output) { - return SerializeImpl(&value_of_default, other_columns, output); -} - } // namespace ROCKSDB_NAMESPACE diff --git a/db/wide/wide_column_serialization_test.cc b/db/wide/wide_column_serialization_test.cc index 8060d2f24..a52d8eb3b 100644 --- a/db/wide/wide_column_serialization_test.cc +++ b/db/wide/wide_column_serialization_test.cc @@ -124,25 +124,6 @@ TEST(WideColumnSerializationTest, SerializeDeserialize) { } } -TEST(WideColumnSerializationTest, SerializeWithPrepend) { - Slice value_of_default("baz"); - WideColumns other_columns{{"foo", "bar"}, {"hello", "world"}}; - - std::string output; - ASSERT_OK(WideColumnSerialization::Serialize(value_of_default, other_columns, - output)); - - Slice input(output); - - WideColumns deserialized_columns; - ASSERT_OK(WideColumnSerialization::Deserialize(input, deserialized_columns)); - - WideColumns expected_columns{{kDefaultWideColumnName, value_of_default}, - other_columns[0], - other_columns[1]}; - ASSERT_EQ(deserialized_columns, expected_columns); -} - TEST(WideColumnSerializationTest, SerializeDuplicateError) { WideColumns columns{{"foo", "bar"}, {"foo", "baz"}}; std::string output; @@ -151,16 +132,6 @@ TEST(WideColumnSerializationTest, SerializeDuplicateError) { WideColumnSerialization::Serialize(columns, output).IsCorruption()); } -TEST(WideColumnSerializationTest, SerializeWithPrependDuplicateError) { - Slice value_of_default("baz"); - WideColumns other_columns{{kDefaultWideColumnName, "dup"}, {"foo", "bar"}}; - - std::string output; - ASSERT_TRUE(WideColumnSerialization::Serialize(value_of_default, - other_columns, output) - .IsCorruption()); -} - TEST(WideColumnSerializationTest, SerializeOutOfOrderError) { WideColumns columns{{"hello", "world"}, {"foo", "bar"}}; std::string output; diff --git a/db/wide/wide_columns_helper.cc b/db/wide/wide_columns_helper.cc new file mode 100644 index 000000000..cf829ce79 --- /dev/null +++ b/db/wide/wide_columns_helper.cc @@ -0,0 +1,46 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/wide/wide_columns_helper.h" + +#include + +#include "db/wide/wide_column_serialization.h" + +namespace ROCKSDB_NAMESPACE { +void WideColumnsHelper::DumpWideColumns(const WideColumns& columns, + std::ostream& os, bool hex) { + if (columns.empty()) { + return; + } + if (hex) { + os << std::hex; + } + auto it = columns.begin(); + os << *it; + for (++it; it != columns.end(); ++it) { + os << ' ' << *it; + } +} + +Status WideColumnsHelper::DumpSliceAsWideColumns(const Slice& value, + std::ostream& os, bool hex) { + WideColumns columns; + Slice value_copy = value; + const Status s = WideColumnSerialization::Deserialize(value_copy, columns); + if (s.ok()) { + DumpWideColumns(columns, os, hex); + } + return s; +} + +void WideColumnsHelper::SortColumns(WideColumns& columns) { + std::sort(columns.begin(), columns.end(), + [](const WideColumn& lhs, const WideColumn& rhs) { + return lhs.name().compare(rhs.name()) < 0; + }); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/wide/wide_columns_helper.h b/db/wide/wide_columns_helper.h new file mode 100644 index 000000000..a870fae30 --- /dev/null +++ b/db/wide/wide_columns_helper.h @@ -0,0 +1,40 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/wide_columns.h" + +namespace ROCKSDB_NAMESPACE { + +class WideColumnsHelper { + public: + static void DumpWideColumns(const WideColumns& columns, std::ostream& os, + bool hex); + + static Status DumpSliceAsWideColumns(const Slice& value, std::ostream& os, + bool hex); + + static bool HasDefaultColumn(const WideColumns& columns) { + return !columns.empty() && columns.front().name() == kDefaultWideColumnName; + } + + static bool HasDefaultColumnOnly(const WideColumns& columns) { + return columns.size() == 1 && + columns.front().name() == kDefaultWideColumnName; + } + + static const Slice& GetDefaultColumn(const WideColumns& columns) { + assert(HasDefaultColumn(columns)); + return columns.front().value(); + } + + static void SortColumns(WideColumns& columns); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/wide/wide_columns_helper_test.cc b/db/wide/wide_columns_helper_test.cc new file mode 100644 index 000000000..482bba531 --- /dev/null +++ b/db/wide/wide_columns_helper_test.cc @@ -0,0 +1,39 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/wide/wide_columns_helper.h" + +#include "db/wide/wide_column_serialization.h" +#include "test_util/testharness.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +TEST(WideColumnsHelperTest, DumpWideColumns) { + WideColumns columns{{"foo", "bar"}, {"hello", "world"}}; + std::ostringstream oss; + WideColumnsHelper::DumpWideColumns(columns, oss, false /* hex */); + EXPECT_EQ("foo:bar hello:world", oss.str()); +} + +TEST(WideColumnsHelperTest, DumpSliceAsWideColumns) { + WideColumns columns{{"foo", "bar"}, {"hello", "world"}}; + std::string output; + ASSERT_OK(WideColumnSerialization::Serialize(columns, output)); + Slice input(output); + + std::ostringstream oss; + ASSERT_OK( + WideColumnsHelper::DumpSliceAsWideColumns(input, oss, false /* hex */)); + + EXPECT_EQ("foo:bar hello:world", oss.str()); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/db/write_batch.cc b/db/write_batch.cc index 47e34e6f9..75f6e1eb4 100644 --- a/db/write_batch.cc +++ b/db/write_batch.cc @@ -39,6 +39,7 @@ #include "rocksdb/write_batch.h" #include +#include #include #include #include @@ -57,6 +58,7 @@ #include "db/snapshot_impl.h" #include "db/trim_history_scheduler.h" #include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "db/write_batch_internal.h" #include "monitoring/perf_context_imp.h" #include "monitoring/statistics_impl.h" @@ -231,9 +233,9 @@ WriteBatch& WriteBatch::operator=(WriteBatch&& src) { return *this; } -WriteBatch::~WriteBatch() {} +WriteBatch::~WriteBatch() = default; -WriteBatch::Handler::~Handler() {} +WriteBatch::Handler::~Handler() = default; void WriteBatch::Handler::LogData(const Slice& /*blob*/) { // If the user has not specified something to do with blobs, then we ignore @@ -739,13 +741,18 @@ SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) { } void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) { - EncodeFixed64(&b->rep_[0], seq); + EncodeFixed64(b->rep_.data(), seq); } size_t WriteBatchInternal::GetFirstOffset(WriteBatch* /*b*/) { return WriteBatchInternal::kHeader; } +void WriteBatchInternal::SetDefaultColumnFamilyTimestampSize( + WriteBatch* wb, size_t default_cf_ts_sz) { + wb->default_cf_ts_sz_ = default_cf_ts_sz; +} + std::tuple WriteBatchInternal::GetColumnFamilyIdAndTimestampSize( WriteBatch* b, ColumnFamilyHandle* column_family) { @@ -943,10 +950,7 @@ Status WriteBatchInternal::PutEntity(WriteBatch* b, uint32_t column_family_id, } WideColumns sorted_columns(columns); - std::sort(sorted_columns.begin(), sorted_columns.end(), - [](const WideColumn& lhs, const WideColumn& rhs) { - return lhs.name().compare(rhs.name()) < 0; - }); + WideColumnsHelper::SortColumns(sorted_columns); std::string entity; const Status s = WideColumnSerialization::Serialize(sorted_columns, entity); @@ -1013,6 +1017,22 @@ Status WriteBatch::PutEntity(ColumnFamilyHandle* column_family, return WriteBatchInternal::PutEntity(this, cf_id, key, columns); } +Status WriteBatch::PutEntity(const Slice& key, + const AttributeGroups& attribute_groups) { + if (attribute_groups.empty()) { + return Status::InvalidArgument( + "Cannot call this method with empty attribute groups"); + } + Status s; + for (const AttributeGroup& ag : attribute_groups) { + s = PutEntity(ag.column_family(), key, ag.columns()); + if (!s.ok()) { + return s; + } + } + return s; +} + Status WriteBatchInternal::InsertNoop(WriteBatch* b) { b->rep_.push_back(static_cast(kTypeNoop)); return Status::OK(); @@ -1836,7 +1856,9 @@ class MemTableInserter : public WriteBatch::Handler { } void DecrementProtectionInfoIdxForTryAgain() { - if (prot_info_ != nullptr) --prot_info_idx_; + if (prot_info_ != nullptr) { + --prot_info_idx_; + } } void ResetProtectionInfo() { @@ -2480,13 +2502,15 @@ class MemTableInserter : public WriteBatch::Handler { } if (perform_merge) { - // 1) Get the existing value - std::string get_value; + // 1) Get the existing value. Use the wide column APIs to make sure we + // don't lose any columns in the process. + PinnableWideColumns existing; // Pass in the sequence number so that we also include previous merge // operations in the same batch. SnapshotImpl read_from_snapshot; read_from_snapshot.number_ = sequence_; + // TODO: plumb Env::IOActivity ReadOptions read_options; read_options.snapshot = &read_from_snapshot; @@ -2495,26 +2519,45 @@ class MemTableInserter : public WriteBatch::Handler { if (cf_handle == nullptr) { cf_handle = db_->DefaultColumnFamily(); } - Status get_status = db_->Get(read_options, cf_handle, key, &get_value); + + Status get_status = + db_->GetEntity(read_options, cf_handle, key, &existing); if (!get_status.ok()) { // Failed to read a key we know exists. Store the delta in memtable. perform_merge = false; } else { - Slice get_value_slice = Slice(get_value); - // 2) Apply this merge auto merge_operator = moptions->merge_operator; assert(merge_operator); + const auto& columns = existing.columns(); + + Status merge_status; std::string new_value; - // `op_failure_scope` (an output parameter) is not provided (set to - // nullptr) since a failure must be propagated regardless of its value. - Status merge_status = MergeHelper::TimedFullMerge( - merge_operator, key, &get_value_slice, {value}, &new_value, - moptions->info_log, moptions->statistics, - SystemClock::Default().get(), /* result_operand */ nullptr, - /* update_num_ops_stats */ false, - /* op_failure_scope */ nullptr); + ValueType new_value_type; + + if (WideColumnsHelper::HasDefaultColumnOnly(columns)) { + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its + // value. + merge_status = MergeHelper::TimedFullMerge( + merge_operator, key, MergeHelper::kPlainBaseValue, + WideColumnsHelper::GetDefaultColumn(columns), {value}, + moptions->info_log, moptions->statistics, + SystemClock::Default().get(), + /* update_num_ops_stats */ false, /* op_failure_scope */ nullptr, + &new_value, /* result_operand */ nullptr, &new_value_type); + } else { + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its + // value. + merge_status = MergeHelper::TimedFullMerge( + merge_operator, key, MergeHelper::kWideBaseValue, columns, + {value}, moptions->info_log, moptions->statistics, + SystemClock::Default().get(), + /* update_num_ops_stats */ false, /* op_failure_scope */ nullptr, + &new_value, /* result_operand */ nullptr, &new_value_type); + } if (!merge_status.ok()) { // Failed to merge! @@ -2523,15 +2566,18 @@ class MemTableInserter : public WriteBatch::Handler { } else { // 3) Add value to memtable assert(!concurrent_memtable_writes_); + assert(new_value_type == kTypeValue || + new_value_type == kTypeWideColumnEntity); + if (kv_prot_info != nullptr) { auto merged_kv_prot_info = kv_prot_info->StripC(column_family_id).ProtectS(sequence_); merged_kv_prot_info.UpdateV(value, new_value); - merged_kv_prot_info.UpdateO(kTypeMerge, kTypeValue); - ret_status = mem->Add(sequence_, kTypeValue, key, new_value, + merged_kv_prot_info.UpdateO(kTypeMerge, new_value_type); + ret_status = mem->Add(sequence_, new_value_type, key, new_value, &merged_kv_prot_info); } else { - ret_status = mem->Add(sequence_, kTypeValue, key, new_value, + ret_status = mem->Add(sequence_, new_value_type, key, new_value, nullptr /* kv_prot_info */); } } @@ -2972,7 +3018,7 @@ class ProtectionInfoUpdater : public WriteBatch::Handler { explicit ProtectionInfoUpdater(WriteBatch::ProtectionInfo* prot_info) : prot_info_(prot_info) {} - ~ProtectionInfoUpdater() override {} + ~ProtectionInfoUpdater() override = default; Status PutCF(uint32_t cf, const Slice& key, const Slice& val) override { return UpdateProtInfo(cf, key, val, kTypeValue); diff --git a/db/write_batch_internal.h b/db/write_batch_internal.h index 1be0bd140..52bbe4545 100644 --- a/db/write_batch_internal.h +++ b/db/write_batch_internal.h @@ -224,6 +224,9 @@ class WriteBatchInternal { static void SetAsLatestPersistentState(WriteBatch* b); static bool IsLatestPersistentState(const WriteBatch* b); + static void SetDefaultColumnFamilyTimestampSize(WriteBatch* wb, + size_t default_cf_ts_sz); + static std::tuple GetColumnFamilyIdAndTimestampSize( WriteBatch* b, ColumnFamilyHandle* column_family); diff --git a/db/write_batch_test.cc b/db/write_batch_test.cc index 174052644..00faea4ce 100644 --- a/db/write_batch_test.cc +++ b/db/write_batch_test.cc @@ -12,7 +12,9 @@ #include "db/column_family.h" #include "db/db_test_util.h" #include "db/memtable.h" +#include "db/wide/wide_columns_helper.h" #include "db/write_batch_internal.h" +#include "dbformat.h" #include "rocksdb/comparator.h" #include "rocksdb/db.h" #include "rocksdb/env.h" @@ -276,6 +278,21 @@ struct TestHandler : public WriteBatch::Handler { } return Status::OK(); } + Status PutEntityCF(uint32_t column_family_id, const Slice& key, + const Slice& entity) override { + std::ostringstream oss; + Status s = WideColumnsHelper::DumpSliceAsWideColumns(entity, oss, false); + if (!s.ok()) { + return s; + } + if (column_family_id == 0) { + seen += "PutEntity(" + key.ToString() + ", " + oss.str() + ")"; + } else { + seen += "PutEntityCF(" + std::to_string(column_family_id) + ", " + + key.ToString() + ", " + oss.str() + ")"; + } + return Status::OK(); + } Status DeleteCF(uint32_t column_family_id, const Slice& key) override { if (column_family_id == 0) { seen += "Delete(" + key.ToString() + ")"; @@ -665,6 +682,82 @@ class ColumnFamilyHandleImplDummy : public ColumnFamilyHandleImpl { }; } // anonymous namespace +TEST_F(WriteBatchTest, AttributeGroupTest) { + WriteBatch batch; + ColumnFamilyHandleImplDummy zero(0), two(2); + AttributeGroups foo_ags; + WideColumn zero_col_1{"0_c_1_n", "0_c_1_v"}; + WideColumn zero_col_2{"0_c_2_n", "0_c_2_v"}; + WideColumns zero_col_1_col_2{zero_col_1, zero_col_2}; + + WideColumn two_col_1{"2_c_1_n", "2_c_1_v"}; + WideColumn two_col_2{"2_c_2_n", "2_c_2_v"}; + WideColumns two_col_1_col_2{two_col_1, two_col_2}; + + foo_ags.emplace_back(&zero, zero_col_1_col_2); + foo_ags.emplace_back(&two, two_col_1_col_2); + + ASSERT_OK(batch.PutEntity("foo", foo_ags)); + + TestHandler handler; + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ( + "PutEntity(foo, 0_c_1_n:0_c_1_v " + "0_c_2_n:0_c_2_v)" + "PutEntityCF(2, foo, 2_c_1_n:2_c_1_v " + "2_c_2_n:2_c_2_v)", + handler.seen); +} + +TEST_F(WriteBatchTest, AttributeGroupSavePointTest) { + WriteBatch batch; + batch.SetSavePoint(); + + ColumnFamilyHandleImplDummy zero(0), two(2), three(3); + AttributeGroups foo_ags; + WideColumn zero_col_1{"0_c_1_n", "0_c_1_v"}; + WideColumn zero_col_2{"0_c_2_n", "0_c_2_v"}; + WideColumns zero_col_1_col_2{zero_col_1, zero_col_2}; + + WideColumn two_col_1{"2_c_1_n", "2_c_1_v"}; + WideColumn two_col_2{"2_c_2_n", "2_c_2_v"}; + WideColumns two_col_1_col_2{two_col_1, two_col_2}; + + foo_ags.emplace_back(&zero, zero_col_1_col_2); + foo_ags.emplace_back(&two, two_col_1_col_2); + + AttributeGroups bar_ags; + WideColumn three_col_1{"3_c_1_n", "3_c_1_v"}; + WideColumn three_col_2{"3_c_2_n", "3_c_2_v"}; + WideColumns three_col_1_col_2{three_col_1, three_col_2}; + + bar_ags.emplace_back(&zero, zero_col_1_col_2); + bar_ags.emplace_back(&three, three_col_1_col_2); + + ASSERT_OK(batch.PutEntity("foo", foo_ags)); + batch.SetSavePoint(); + + ASSERT_OK(batch.PutEntity("bar", bar_ags)); + + TestHandler handler; + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ( + "PutEntity(foo, 0_c_1_n:0_c_1_v 0_c_2_n:0_c_2_v)" + "PutEntityCF(2, foo, 2_c_1_n:2_c_1_v 2_c_2_n:2_c_2_v)" + "PutEntity(bar, 0_c_1_n:0_c_1_v 0_c_2_n:0_c_2_v)" + "PutEntityCF(3, bar, 3_c_1_n:3_c_1_v 3_c_2_n:3_c_2_v)", + handler.seen); + + ASSERT_OK(batch.RollbackToSavePoint()); + + handler.seen.clear(); + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ( + "PutEntity(foo, 0_c_1_n:0_c_1_v 0_c_2_n:0_c_2_v)" + "PutEntityCF(2, foo, 2_c_1_n:2_c_1_v 2_c_2_n:2_c_2_v)", + handler.seen); +} + TEST_F(WriteBatchTest, ColumnFamiliesBatchTest) { WriteBatch batch; ColumnFamilyHandleImplDummy zero(0), two(2), three(3), eight(8); @@ -677,6 +770,9 @@ TEST_F(WriteBatchTest, ColumnFamiliesBatchTest) { ASSERT_OK(batch.Merge(&three, Slice("threethree"), Slice("3three"))); ASSERT_OK(batch.Put(&zero, Slice("foo"), Slice("bar"))); ASSERT_OK(batch.Merge(Slice("omom"), Slice("nom"))); + // TODO(yuzhangyu): implement this. + ASSERT_TRUE( + batch.TimedPut(&zero, Slice("foo"), Slice("bar"), 0u).IsNotSupported()); TestHandler handler; ASSERT_OK(batch.Iterate(&handler)); @@ -704,6 +800,8 @@ TEST_F(WriteBatchTest, ColumnFamiliesBatchWithIndexTest) { ASSERT_OK(batch.Merge(&three, Slice("threethree"), Slice("3three"))); ASSERT_OK(batch.Put(&zero, Slice("foo"), Slice("bar"))); ASSERT_OK(batch.Merge(Slice("omom"), Slice("nom"))); + ASSERT_TRUE( + batch.TimedPut(&zero, Slice("foo"), Slice("bar"), 0u).IsNotSupported()); std::unique_ptr iter; diff --git a/db/write_callback_test.cc b/db/write_callback_test.cc index 1be8593f1..7709257f0 100644 --- a/db/write_callback_test.cc +++ b/db/write_callback_test.cc @@ -64,7 +64,7 @@ class MockWriteCallback : public WriteCallback { bool allow_batching_ = false; std::atomic was_called_{false}; - MockWriteCallback() {} + MockWriteCallback() = default; MockWriteCallback(const MockWriteCallback& other) { should_fail_ = other.should_fail_; @@ -111,7 +111,7 @@ TEST_P(WriteCallbackPTest, WriteWithCallbackTest) { WriteOP(bool should_fail = false) { callback_.should_fail_ = should_fail; } void Put(const string& key, const string& val) { - kvs_.push_back(std::make_pair(key, val)); + kvs_.emplace_back(key, val); ASSERT_OK(write_batch_.Put(key, val)); } @@ -176,8 +176,7 @@ TEST_P(WriteCallbackPTest, WriteWithCallbackTest) { DBOptions db_options(options); ColumnFamilyOptions cf_options(options); std::vector column_families; - column_families.push_back( - ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + column_families.emplace_back(kDefaultColumnFamilyName, cf_options); std::vector handles; auto open_s = DBImpl::Open(db_options, dbname, column_families, &handles, &db, seq_per_batch_, true /* batch_per_txn */); diff --git a/db_stress_tool/CMakeLists.txt b/db_stress_tool/CMakeLists.txt index 51d6ea0d6..60c02e173 100644 --- a/db_stress_tool/CMakeLists.txt +++ b/db_stress_tool/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(db_stress${ARTIFACT_SUFFIX} db_stress_shared_state.cc db_stress_stat.cc db_stress_test_base.cc + db_stress_wide_merge_operator.cc db_stress_tool.cc expected_state.cc expected_value.cc diff --git a/db_stress_tool/batched_ops_stress.cc b/db_stress_tool/batched_ops_stress.cc index 0872f2842..25e4d2372 100644 --- a/db_stress_tool/batched_ops_stress.cc +++ b/db_stress_tool/batched_ops_stress.cc @@ -13,8 +13,8 @@ namespace ROCKSDB_NAMESPACE { class BatchedOpsStressTest : public StressTest { public: - BatchedOpsStressTest() {} - virtual ~BatchedOpsStressTest() {} + BatchedOpsStressTest() = default; + virtual ~BatchedOpsStressTest() = default; bool IsStateTracked() const override { return false; } @@ -52,11 +52,11 @@ class BatchedOpsStressTest : public StressTest { const std::string k = num + key_body; const std::string v = value_body + num; - if (FLAGS_use_merge) { - batch.Merge(cfh, k, v); - } else if (FLAGS_use_put_entity_one_in > 0 && - (value_base % FLAGS_use_put_entity_one_in) == 0) { + if (FLAGS_use_put_entity_one_in > 0 && + (value_base % FLAGS_use_put_entity_one_in) == 0) { batch.PutEntity(cfh, k, GenerateWideColumns(value_base, v)); + } else if (FLAGS_use_merge) { + batch.Merge(cfh, k, v); } else { batch.Put(cfh, k, v); } diff --git a/db_stress_tool/cf_consistency_stress.cc b/db_stress_tool/cf_consistency_stress.cc index f3d9b71d9..da382ae3b 100644 --- a/db_stress_tool/cf_consistency_stress.cc +++ b/db_stress_tool/cf_consistency_stress.cc @@ -16,7 +16,7 @@ class CfConsistencyStressTest : public StressTest { public: CfConsistencyStressTest() : batch_id_(0) {} - ~CfConsistencyStressTest() override {} + ~CfConsistencyStressTest() override = default; bool IsStateTracked() const override { return false; } @@ -36,18 +36,15 @@ class CfConsistencyStressTest : public StressTest { WriteBatch batch; - const bool use_put_entity = !FLAGS_use_merge && - FLAGS_use_put_entity_one_in > 0 && - (value_base % FLAGS_use_put_entity_one_in) == 0; - for (auto cf : rand_column_families) { ColumnFamilyHandle* const cfh = column_families_[cf]; assert(cfh); - if (FLAGS_use_merge) { - batch.Merge(cfh, k, v); - } else if (use_put_entity) { + if (FLAGS_use_put_entity_one_in > 0 && + (value_base % FLAGS_use_put_entity_one_in) == 0) { batch.PutEntity(cfh, k, GenerateWideColumns(value_base, v)); + } else if (FLAGS_use_merge) { + batch.Merge(cfh, k, v); } else { batch.Put(cfh, k, v); } @@ -235,7 +232,7 @@ class CfConsistencyStressTest : public StressTest { } db_->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(), statuses.data()); - for (auto s : statuses) { + for (const auto& s : statuses) { if (s.ok()) { // found case thread->stats.AddGets(1, 1); diff --git a/db_stress_tool/db_stress_common.cc b/db_stress_tool/db_stress_common.cc index 93436d0f8..dc600a49f 100644 --- a/db_stress_tool/db_stress_common.cc +++ b/db_stress_tool/db_stress_common.cc @@ -13,6 +13,7 @@ #include +#include "rocksdb/secondary_cache.h" #include "util/file_checksum_helper.h" #include "util/xxhash.h" @@ -21,6 +22,8 @@ ROCKSDB_NAMESPACE::Env* db_stress_env = nullptr; // If non-null, injects read error at a rate specified by the // read_fault_one_in or write_fault_one_in flag std::shared_ptr fault_fs_guard; +std::shared_ptr compressed_secondary_cache; +std::shared_ptr block_cache; enum ROCKSDB_NAMESPACE::CompressionType compression_type_e = ROCKSDB_NAMESPACE::kSnappyCompression; enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e = @@ -148,6 +151,85 @@ void DbVerificationThread(void* v) { } } +void CompressedCacheSetCapacityThread(void* v) { + assert(FLAGS_compressed_secondary_cache_size > 0 || + FLAGS_compressed_secondary_cache_ratio > 0.0); + auto* thread = reinterpret_cast(v); + SharedState* shared = thread->shared; + while (true) { + { + MutexLock l(shared->GetMutex()); + if (shared->ShouldStopBgThread()) { + shared->IncBgThreadsFinished(); + if (shared->BgThreadsFinished()) { + shared->GetCondVar()->SignalAll(); + } + return; + } + } + db_stress_env->SleepForMicroseconds(FLAGS_secondary_cache_update_interval); + if (FLAGS_compressed_secondary_cache_size > 0) { + Status s = compressed_secondary_cache->SetCapacity(0); + size_t capacity; + if (s.ok()) { + s = compressed_secondary_cache->GetCapacity(capacity); + assert(capacity == 0); + } + db_stress_env->SleepForMicroseconds(10 * 1000 * 1000); + if (s.ok()) { + s = compressed_secondary_cache->SetCapacity( + FLAGS_compressed_secondary_cache_size); + } + if (s.ok()) { + s = compressed_secondary_cache->GetCapacity(capacity); + assert(capacity == FLAGS_compressed_secondary_cache_size); + } + if (!s.ok()) { + fprintf(stderr, "Compressed cache Set/GetCapacity returned error: %s\n", + s.ToString().c_str()); + } + } else if (FLAGS_compressed_secondary_cache_ratio > 0.0) { + if (thread->rand.OneIn(2)) { // if (thread->rand.OneIn(2)) { + size_t capacity = block_cache->GetCapacity(); + size_t adjustment; + if (FLAGS_use_write_buffer_manager && FLAGS_db_write_buffer_size > 0) { + adjustment = (capacity - FLAGS_db_write_buffer_size); + } else { + adjustment = capacity; + } + // Lower by upto 50% of usable block cache capacity + adjustment = (adjustment * thread->rand.Uniform(50)) / 100; + block_cache->SetCapacity(capacity - adjustment); + fprintf(stderr, "New cache capacity = %lu\n", + block_cache->GetCapacity()); + db_stress_env->SleepForMicroseconds(10 * 1000 * 1000); + block_cache->SetCapacity(capacity); + } else { + Status s; + double new_comp_cache_ratio = + (double)thread->rand.Uniform( + FLAGS_compressed_secondary_cache_ratio * 100) / + 100; + fprintf(stderr, "New comp cache ratio = %f\n", new_comp_cache_ratio); + + s = UpdateTieredCache(block_cache, /*capacity*/ -1, + new_comp_cache_ratio); + if (s.ok()) { + db_stress_env->SleepForMicroseconds(10 * 1000 * 1000); + } + if (s.ok()) { + s = UpdateTieredCache(block_cache, /*capacity*/ -1, + FLAGS_compressed_secondary_cache_ratio); + } + if (!s.ok()) { + fprintf(stderr, "UpdateTieredCache returned error: %s\n", + s.ToString().c_str()); + } + } + } + } +} + void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz) { if (!FLAGS_verbose) { return; @@ -270,10 +352,7 @@ WideColumns GenerateExpectedWideColumns(uint32_t value_base, WideColumns columns = GenerateWideColumns(value_base, slice); - std::sort(columns.begin(), columns.end(), - [](const WideColumn& lhs, const WideColumn& rhs) { - return lhs.name().compare(rhs.name()) < 0; - }); + WideColumnsHelper::SortColumns(columns); return columns; } @@ -296,15 +375,11 @@ bool VerifyWideColumns(const Slice& value, const WideColumns& columns) { } bool VerifyWideColumns(const WideColumns& columns) { - if (columns.empty()) { - return false; - } - - if (columns.front().name() != kDefaultWideColumnName) { + if (!WideColumnsHelper::HasDefaultColumn(columns)) { return false; } - const Slice& value_of_default = columns.front().value(); + const Slice& value_of_default = WideColumnsHelper::GetDefaultColumn(columns); return VerifyWideColumns(value_of_default, columns); } diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index c0b1e6fd2..adbc554ab 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -37,6 +37,7 @@ #include "db/db_impl/db_impl.h" #include "db/version_set.h" +#include "db/wide/wide_columns_helper.h" #include "db_stress_tool/db_stress_env_wrapper.h" #include "db_stress_tool/db_stress_listener.h" #include "db_stress_tool/db_stress_shared_state.h" @@ -107,11 +108,14 @@ DECLARE_int32(max_write_buffer_number); DECLARE_int32(min_write_buffer_number_to_merge); DECLARE_int32(max_write_buffer_number_to_maintain); DECLARE_int64(max_write_buffer_size_to_maintain); +DECLARE_bool(use_write_buffer_manager); DECLARE_double(memtable_prefix_bloom_size_ratio); DECLARE_bool(memtable_whole_key_filtering); DECLARE_int32(open_files); -DECLARE_int64(compressed_cache_size); -DECLARE_int32(compressed_cache_numshardbits); +DECLARE_uint64(compressed_secondary_cache_size); +DECLARE_int32(compressed_secondary_cache_numshardbits); +DECLARE_int32(secondary_cache_update_interval); +DECLARE_double(compressed_secondary_cache_ratio); DECLARE_int32(compaction_style); DECLARE_int32(compaction_pri); DECLARE_int32(num_levels); @@ -158,7 +162,7 @@ DECLARE_double(experimental_mempurge_threshold); DECLARE_bool(enable_write_thread_adaptive_yield); DECLARE_int32(reopen); DECLARE_double(bloom_bits); -DECLARE_int32(ribbon_starting_level); +DECLARE_int32(bloom_before_level); DECLARE_bool(partition_filters); DECLARE_bool(optimize_filters_for_memory); DECLARE_bool(detect_filter_construct_corruption); @@ -229,6 +233,7 @@ DECLARE_int32(compression_zstd_max_train_bytes); DECLARE_int32(compression_parallel_threads); DECLARE_uint64(compression_max_dict_buffer_bytes); DECLARE_bool(compression_use_zstd_dict_trainer); +DECLARE_bool(compression_checksum); DECLARE_string(checksum_type); DECLARE_string(env_uri); DECLARE_string(fs_uri); @@ -248,10 +253,12 @@ DECLARE_bool(avoid_flush_during_recovery); DECLARE_uint64(max_write_batch_group_size_bytes); DECLARE_bool(level_compaction_dynamic_level_bytes); DECLARE_int32(verify_checksum_one_in); +DECLARE_int32(verify_file_checksums_one_in); DECLARE_int32(verify_db_one_in); DECLARE_int32(continuous_verification_interval); DECLARE_int32(get_property_one_in); DECLARE_string(file_checksum_impl); +DECLARE_bool(verification_only); // Options for transaction dbs. // Use TransactionDB (a.k.a. Pessimistic Transaction DB) @@ -306,6 +313,7 @@ DECLARE_uint32(memtable_protection_bytes_per_key); DECLARE_uint32(block_protection_bytes_per_key); DECLARE_uint64(user_timestamp_size); +DECLARE_bool(persist_user_defined_timestamps); DECLARE_string(secondary_cache_uri); DECLARE_int32(secondary_cache_fault_one_in); @@ -327,6 +335,10 @@ DECLARE_bool(allow_data_in_errors); DECLARE_bool(enable_thread_tracking); +DECLARE_uint32(memtable_max_range_deletions); + +DECLARE_uint32(bottommost_file_compaction_delay); + // Tiered storage DECLARE_bool(enable_tiered_storage); // set last_level_temperature DECLARE_int64(preclude_last_level_data_seconds); @@ -340,6 +352,7 @@ DECLARE_uint64(initial_auto_readahead_size); DECLARE_uint64(max_auto_readahead_size); DECLARE_uint64(num_file_reads_for_auto_readahead); DECLARE_bool(use_io_uring); +DECLARE_bool(auto_readahead_size); constexpr long KB = 1024; constexpr int kRandomValueMaxFactor = 3; @@ -349,6 +362,9 @@ constexpr int kValueMaxLen = 100; extern ROCKSDB_NAMESPACE::Env* db_stress_env; extern ROCKSDB_NAMESPACE::Env* db_stress_listener_env; extern std::shared_ptr fault_fs_guard; +extern std::shared_ptr + compressed_secondary_cache; +extern std::shared_ptr block_cache; extern enum ROCKSDB_NAMESPACE::CompressionType compression_type_e; extern enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e; @@ -621,13 +637,7 @@ inline std::string WideColumnsToHex(const WideColumns& columns) { std::ostringstream oss; - oss << std::hex; - - auto it = columns.begin(); - oss << *it; - for (++it; it != columns.end(); ++it) { - oss << ' ' << *it; - } + WideColumnsHelper::DumpWideColumns(columns, oss, true); return oss.str(); } @@ -647,6 +657,8 @@ extern void PoolSizeChangeThread(void* v); extern void DbVerificationThread(void* v); +extern void CompressedCacheSetCapacityThread(void* v); + extern void TimestampedSnapshotsThread(void* v); extern void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz); diff --git a/db_stress_tool/db_stress_driver.cc b/db_stress_tool/db_stress_driver.cc index 4bf82c9d1..92730beca 100644 --- a/db_stress_tool/db_stress_driver.cc +++ b/db_stress_tool/db_stress_driver.cc @@ -14,6 +14,7 @@ namespace ROCKSDB_NAMESPACE { void ThreadBody(void* v) { + ThreadStatusUtil::RegisterThread(db_stress_env, ThreadStatus::USER); ThreadState* thread = reinterpret_cast(v); SharedState* shared = thread->shared; @@ -26,34 +27,40 @@ void ThreadBody(void* v) { if (shared->AllInitialized()) { shared->GetCondVar()->SignalAll(); } - while (!shared->Started()) { - shared->GetCondVar()->Wait(); - } } - thread->shared->GetStressTest()->OperateDb(thread); - - { - MutexLock l(shared->GetMutex()); - shared->IncOperated(); - if (shared->AllOperated()) { - shared->GetCondVar()->SignalAll(); + if (!FLAGS_verification_only) { + { + MutexLock l(shared->GetMutex()); + while (!shared->Started()) { + shared->GetCondVar()->Wait(); + } } - while (!shared->VerifyStarted()) { - shared->GetCondVar()->Wait(); + thread->shared->GetStressTest()->OperateDb(thread); + { + MutexLock l(shared->GetMutex()); + shared->IncOperated(); + if (shared->AllOperated()) { + shared->GetCondVar()->SignalAll(); + } + while (!shared->VerifyStarted()) { + shared->GetCondVar()->Wait(); + } } - } - if (!FLAGS_skip_verifydb) { - thread->shared->GetStressTest()->VerifyDb(thread); - } + if (!FLAGS_skip_verifydb) { + thread->shared->GetStressTest()->VerifyDb(thread); + } - { - MutexLock l(shared->GetMutex()); - shared->IncDone(); - if (shared->AllDone()) { - shared->GetCondVar()->SignalAll(); + { + MutexLock l(shared->GetMutex()); + shared->IncDone(); + if (shared->AllDone()) { + shared->GetCondVar()->SignalAll(); + } } } + + ThreadStatusUtil::UnregisterThread(); } bool RunStressTestImpl(SharedState* shared) { SystemClock* clock = db_stress_env->GetSystemClock().get(); @@ -74,12 +81,30 @@ bool RunStressTestImpl(SharedState* shared) { stress->InitDb(shared); stress->FinishInitDb(shared); - if (FLAGS_sync_fault_injection) { - fault_fs_guard->SetFilesystemDirectWritable(false); - } if (FLAGS_write_fault_one_in) { + if (!FLAGS_sync_fault_injection) { + // unsynced WAL loss is not supported without sync_fault_injection + fault_fs_guard->SetDirectWritableTypes({kWalFile}); + } + IOStatus error_msg; + if (FLAGS_inject_error_severity <= 1 || FLAGS_inject_error_severity > 2) { + error_msg = IOStatus::IOError("Retryable injected write error"); + error_msg.SetRetryable(true); + } else if (FLAGS_inject_error_severity == 2) { + error_msg = IOStatus::IOError("Fatal injected write error"); + error_msg.SetDataLoss(true); + } + // TODO: inject write error for other file types including + // MANIFEST, CURRENT, and WAL files. + fault_fs_guard->SetRandomWriteError( + shared->GetSeed(), FLAGS_write_fault_one_in, error_msg, + /*inject_for_all_file_types=*/false, {FileType::kTableFile}); + fault_fs_guard->SetFilesystemDirectWritable(false); fault_fs_guard->EnableWriteErrorInjection(); } + if (FLAGS_sync_fault_injection) { + fault_fs_guard->SetFilesystemDirectWritable(false); + } uint32_t n = FLAGS_threads; uint64_t now = clock->NowMicros(); @@ -96,6 +121,11 @@ bool RunStressTestImpl(SharedState* shared) { shared->IncBgThreads(); } + if (FLAGS_compressed_secondary_cache_size > 0 || + FLAGS_compressed_secondary_cache_ratio > 0.0) { + shared->IncBgThreads(); + } + std::vector threads(n); for (uint32_t i = 0; i < n; i++) { threads[i] = new ThreadState(i, shared); @@ -113,6 +143,13 @@ bool RunStressTestImpl(SharedState* shared) { &continuous_verification_thread); } + ThreadState compressed_cache_set_capacity_thread(0, shared); + if (FLAGS_compressed_secondary_cache_size > 0 || + FLAGS_compressed_secondary_cache_ratio > 0.0) { + db_stress_env->StartThread(CompressedCacheSetCapacityThread, + &compressed_cache_set_capacity_thread); + } + // Each thread goes through the following states: // initializing -> wait for others to init -> read/populate/depopulate // wait for others to operate -> verify -> done @@ -139,45 +176,55 @@ bool RunStressTestImpl(SharedState* shared) { } } - // This is after the verification step to avoid making all those `Get()`s - // and `MultiGet()`s contend on the DB-wide trace mutex. - if (!FLAGS_expected_values_dir.empty()) { - stress->TrackExpectedState(shared); - } - - now = clock->NowMicros(); - fprintf(stdout, "%s Starting database operations\n", - clock->TimeToString(now / 1000000).c_str()); + if (!FLAGS_verification_only) { + // This is after the verification step to avoid making all those `Get()`s + // and `MultiGet()`s contend on the DB-wide trace mutex. + if (!FLAGS_expected_values_dir.empty()) { + stress->TrackExpectedState(shared); + } + now = clock->NowMicros(); + fprintf(stdout, "%s Starting database operations\n", + clock->TimeToString(now / 1000000).c_str()); - shared->SetStart(); - shared->GetCondVar()->SignalAll(); - while (!shared->AllOperated()) { - shared->GetCondVar()->Wait(); - } + shared->SetStart(); + shared->GetCondVar()->SignalAll(); + while (!shared->AllOperated()) { + shared->GetCondVar()->Wait(); + } - now = clock->NowMicros(); - if (FLAGS_test_batches_snapshots) { - fprintf(stdout, "%s Limited verification already done during gets\n", - clock->TimeToString((uint64_t)now / 1000000).c_str()); - } else if (FLAGS_skip_verifydb) { - fprintf(stdout, "%s Verification skipped\n", - clock->TimeToString((uint64_t)now / 1000000).c_str()); - } else { - fprintf(stdout, "%s Starting verification\n", - clock->TimeToString((uint64_t)now / 1000000).c_str()); - } + now = clock->NowMicros(); + if (FLAGS_test_batches_snapshots) { + fprintf(stdout, "%s Limited verification already done during gets\n", + clock->TimeToString((uint64_t)now / 1000000).c_str()); + } else if (FLAGS_skip_verifydb) { + fprintf(stdout, "%s Verification skipped\n", + clock->TimeToString((uint64_t)now / 1000000).c_str()); + } else { + fprintf(stdout, "%s Starting verification\n", + clock->TimeToString((uint64_t)now / 1000000).c_str()); + } - shared->SetStartVerify(); - shared->GetCondVar()->SignalAll(); - while (!shared->AllDone()) { - shared->GetCondVar()->Wait(); + shared->SetStartVerify(); + shared->GetCondVar()->SignalAll(); + while (!shared->AllDone()) { + shared->GetCondVar()->Wait(); + } } } - for (unsigned int i = 1; i < n; i++) { - threads[0]->stats.Merge(threads[i]->stats); + // If we are running verification_only + // stats will be empty and trying to report them will + // emit no ops or writes error. To avoid this, merging and reporting stats + // are not executed when running with verification_only + // TODO: We need to create verification stats (e.g. how many keys + // are verified by which method) and report them here instead of operation + // stats. + if (!FLAGS_verification_only) { + for (unsigned int i = 1; i < n; i++) { + threads[0]->stats.Merge(threads[i]->stats); + } + threads[0]->stats.Report("Stress Test"); } - threads[0]->stats.Report("Stress Test"); for (unsigned int i = 0; i < n; i++) { delete threads[i]; @@ -189,10 +236,15 @@ bool RunStressTestImpl(SharedState* shared) { fprintf(stdout, "%s Verification successful\n", clock->TimeToString(now / 1000000).c_str()); } - stress->PrintStatistics(); + + if (!FLAGS_verification_only) { + stress->PrintStatistics(); + } if (FLAGS_compaction_thread_pool_adjust_interval > 0 || - FLAGS_continuous_verification_interval > 0) { + FLAGS_continuous_verification_interval > 0 || + FLAGS_compressed_secondary_cache_size > 0 || + FLAGS_compressed_secondary_cache_ratio > 0.0) { MutexLock l(shared->GetMutex()); shared->SetShouldStopBgThread(); while (!shared->BgThreadsFinished()) { diff --git a/db_stress_tool/db_stress_env_wrapper.h b/db_stress_tool/db_stress_env_wrapper.h index 612d9fc6b..83e6838c7 100644 --- a/db_stress_tool/db_stress_env_wrapper.h +++ b/db_stress_tool/db_stress_env_wrapper.h @@ -32,6 +32,48 @@ class DbStressRandomAccessFileWrapper : public FSRandomAccessFileOwnerWrapper { #endif return target()->Read(offset, n, options, result, scratch, dbg); } + + IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs, + const IOOptions& options, IODebugContext* dbg) override { +#ifndef NDEBUG + const ThreadStatus::OperationType thread_op = + ThreadStatusUtil::GetThreadOperation(); + Env::IOActivity io_activity = + ThreadStatusUtil::TEST_GetExpectedIOActivity(thread_op); + assert(io_activity == Env::IOActivity::kUnknown || + io_activity == options.io_activity); +#endif + return target()->MultiRead(reqs, num_reqs, options, dbg); + } + + IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options, + IODebugContext* dbg) override { +#ifndef NDEBUG + const ThreadStatus::OperationType thread_op = + ThreadStatusUtil::GetThreadOperation(); + Env::IOActivity io_activity = + ThreadStatusUtil::TEST_GetExpectedIOActivity(thread_op); + assert(io_activity == Env::IOActivity::kUnknown || + io_activity == options.io_activity); +#endif + return target()->Prefetch(offset, n, options, dbg); + } + + IOStatus ReadAsync(FSReadRequest& req, const IOOptions& options, + std::function cb, + void* cb_arg, void** io_handle, IOHandleDeleter* del_fn, + IODebugContext* dbg) override { +#ifndef NDEBUG + const ThreadStatus::OperationType thread_op = + ThreadStatusUtil::GetThreadOperation(); + Env::IOActivity io_activity = + ThreadStatusUtil::TEST_GetExpectedIOActivity(thread_op); + assert(io_activity == Env::IOActivity::kUnknown || + io_activity == options.io_activity); +#endif + return target()->ReadAsync(req, options, cb, cb_arg, io_handle, del_fn, + dbg); + } }; class DbStressFSWrapper : public FileSystemWrapper { diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index df4c3be0b..c6ffbc93e 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -136,6 +136,9 @@ DEFINE_uint64(db_write_buffer_size, ROCKSDB_NAMESPACE::Options().db_write_buffer_size, "Number of bytes to buffer in all memtables before compacting"); +DEFINE_bool(use_write_buffer_manager, false, + "Charge WriteBufferManager memory to the block cache"); + DEFINE_int32( write_buffer_size, static_cast(ROCKSDB_NAMESPACE::Options().write_buffer_size), @@ -198,15 +201,23 @@ DEFINE_int32(open_files, ROCKSDB_NAMESPACE::Options().max_open_files, "Maximum number of files to keep open at the same time " "(use default if == 0)"); -DEFINE_int64(compressed_cache_size, 0, - "Number of bytes to use as a cache of compressed data." - " 0 means use default settings."); +DEFINE_uint64(compressed_secondary_cache_size, 0, + "Number of bytes to use as a cache of compressed data." + " 0 means use default settings."); -DEFINE_int32( - compressed_cache_numshardbits, -1, - "Number of shards for the compressed block cache is 2 ** " - "compressed_cache_numshardbits. Negative value means default settings. " - "This is applied only if compressed_cache_size is greater than 0."); +DEFINE_int32(compressed_secondary_cache_numshardbits, -1, + "Number of shards for the compressed secondary cache is 2 ** " + "compressed_secondary_cache_numshardbits. " + "Negative value means default settings. This is applied only " + "if compressed_secondary_cache_size is greater than 0."); + +DEFINE_double(compressed_secondary_cache_ratio, 0.0, + "Fraction of block cache memory budget to use for compressed " + "secondary cache"); + +DEFINE_int32(secondary_cache_update_interval, 30 * 1000 * 1000, + "Interval between modification of secondary cache parameters, in " + "microseconds"); DEFINE_int32(compaction_style, ROCKSDB_NAMESPACE::Options().compaction_style, ""); @@ -526,10 +537,11 @@ DEFINE_double(bloom_bits, 10, "Negative means use default settings."); DEFINE_int32( - ribbon_starting_level, 999, + bloom_before_level, 999, "Use Bloom filter on levels below specified and Ribbon beginning on level " - "specified. Flush is considered level -1. 999 or more -> always Bloom. 0 " - "-> Ribbon except Bloom for flush. -1 -> always Ribbon."); + "specified. Flush is considered level -1. Setting -1 -> always Ribbon. " + "0 -> Ribbon except Bloom for flush. INT_MAX (typically 2147483647) -> " + "always Bloom."); DEFINE_bool(partition_filters, false, "use partitioned filters " @@ -845,6 +857,9 @@ DEFINE_bool( "ZSTD 1.4.5+ is required. If ZSTD 1.4.5+ is not linked with the binary, " "this flag will have the default value true."); +DEFINE_bool(compression_checksum, false, + "Turn on zstd's checksum feature for detecting corruption."); + DEFINE_string(bottommost_compression_type, "disable", "Algorithm to use to compress bottommost level of the database. " "\"disable\" means disabling the feature"); @@ -929,6 +944,13 @@ DEFINE_int32(verify_checksum_one_in, 0, " checksum verification of all the files in the database once for" " every N ops on average. 0 indicates that calls to" " VerifyChecksum() are disabled."); + +DEFINE_int32(verify_file_checksums_one_in, 0, + "If non-zero, then DB::VerifyFileChecksums() will be called to do" + " checksum verification of all the files in the database once for" + " every N ops on average. 0 indicates that calls to" + " VerifyFileChecksums() are disabled."); + DEFINE_int32(verify_db_one_in, 0, "If non-zero, call VerifyDb() once for every N ops. 0 indicates " "that VerifyDb() will not be called in OperateDb(). Note that " @@ -996,12 +1018,17 @@ DEFINE_string(file_checksum_impl, "none", "\"none\" for null."); DEFINE_int32(write_fault_one_in, 0, - "On non-zero, enables fault injection on write"); + "On non-zero, enables fault injection on write. Currently only" + "injects write error when writing to SST files."); DEFINE_uint64(user_timestamp_size, 0, "Number of bytes for a user-defined timestamp. Currently, only " "8-byte is supported"); +DEFINE_bool(persist_user_defined_timestamps, true, + "Flag to indicate whether user-defined timestamps will be persisted" + " during Flush"); + DEFINE_int32(open_metadata_write_fault_one_in, 0, "On non-zero, enables fault injection on file metadata write " "during DB reopen."); @@ -1011,14 +1038,17 @@ DEFINE_string(secondary_cache_uri, "", DEFINE_int32(secondary_cache_fault_one_in, 0, "On non-zero, enables fault injection in secondary cache inserts" " and lookups"); +DEFINE_double(tiered_cache_percent_compressed, 0.0, + "Percentage of total block cache budget to allocate to the " + "compressed cache"); DEFINE_int32(open_write_fault_one_in, 0, "On non-zero, enables fault injection on file writes " "during DB reopen."); DEFINE_int32(open_read_fault_one_in, 0, "On non-zero, enables fault injection on file reads " "during DB reopen."); -DEFINE_int32(injest_error_severity, 1, - "The severity of the injested IO Error. 1 is soft error (e.g. " +DEFINE_int32(inject_error_severity, 1, + "The severity of the injected IO Error. 1 is soft error (e.g. " "retryable error), 2 is fatal error, and the default is " "retryable error."); DEFINE_int32(prepopulate_block_cache, @@ -1100,6 +1130,20 @@ DEFINE_uint64(stats_dump_period_sec, "Gap between printing stats to log in seconds"); DEFINE_bool(use_io_uring, false, "Enable the use of IO uring on Posix"); + +DEFINE_bool(verification_only, false, + "If true, tests will only execute verification step"); extern "C" bool RocksDbIOUringEnable() { return FLAGS_use_io_uring; } +DEFINE_uint32(memtable_max_range_deletions, 0, + "If nonzero, RocksDB will try to flush the current memtable" + "after the number of range deletions is >= this limit"); + +DEFINE_uint32(bottommost_file_compaction_delay, 0, + "Delay kBottommostFiles compaction by this amount of seconds." + "See more in option comment."); + +DEFINE_bool(auto_readahead_size, false, + "Does auto tuning of readahead_size when enabled during scans."); + #endif // GFLAGS diff --git a/db_stress_tool/db_stress_listener.cc b/db_stress_tool/db_stress_listener.cc index e2838c582..64adca877 100644 --- a/db_stress_tool/db_stress_listener.cc +++ b/db_stress_tool/db_stress_listener.cc @@ -67,7 +67,7 @@ UniqueIdVerifier::UniqueIdVerifier(const std::string& db_name, Env* env) std::string id(24U, '\0'); Slice result; for (;;) { - s = reader->Read(id.size(), opts, &result, &id[0], /*dbg*/ nullptr); + s = reader->Read(id.size(), opts, &result, id.data(), /*dbg*/ nullptr); if (!s.ok()) { fprintf(stderr, "Error reading unique id file: %s\n", s.ToString().c_str()); diff --git a/db_stress_tool/db_stress_listener.h b/db_stress_tool/db_stress_listener.h index 97bbdaefa..505b0a604 100644 --- a/db_stress_tool/db_stress_listener.h +++ b/db_stress_tool/db_stress_listener.h @@ -9,6 +9,7 @@ #include #include +#include "db_stress_tool/db_stress_shared_state.h" #include "file/filename.h" #include "file/writable_file_writer.h" #include "rocksdb/db.h" @@ -19,9 +20,12 @@ #include "rocksdb/unique_id.h" #include "util/gflags_compat.h" #include "util/random.h" +#include "utilities/fault_injection_fs.h" DECLARE_int32(compact_files_one_in); +extern std::shared_ptr fault_fs_guard; + namespace ROCKSDB_NAMESPACE { // Verify across process executions that all seen IDs are unique @@ -67,11 +71,23 @@ class DbStressListener : public EventListener { VerifyFilePath(info.file_path); // pretending doing some work here RandomSleep(); + if (FLAGS_read_fault_one_in) { + (void)fault_fs_guard->GetAndResetErrorCount(); + fault_fs_guard->DisableErrorInjection(); + } } void OnFlushBegin(DB* /*db*/, const FlushJobInfo& /*flush_job_info*/) override { RandomSleep(); + if (FLAGS_read_fault_one_in) { + // Hardcoded to inject retryable error as a non-retryable error would put + // the DB in read-only mode and then it would crash on the next write. + fault_fs_guard->SetThreadLocalReadErrorContext( + static_cast(FLAGS_seed), FLAGS_read_fault_one_in, + true /* retryable */); + fault_fs_guard->EnableErrorInjection(); + } } void OnTableFileDeleted(const TableFileDeletionInfo& /*info*/) override { @@ -95,6 +111,24 @@ class DbStressListener : public EventListener { RandomSleep(); } + void OnSubcompactionBegin(const SubcompactionJobInfo& /* si */) override { + if (FLAGS_read_fault_one_in) { + // Hardcoded to inject retryable error as a non-retryable error would put + // the DB in read-only mode and then it would crash on the next write. + fault_fs_guard->SetThreadLocalReadErrorContext( + static_cast(FLAGS_seed), FLAGS_read_fault_one_in, + true /* retryable */); + fault_fs_guard->EnableErrorInjection(); + } + } + + void OnSubcompactionCompleted(const SubcompactionJobInfo& /* si */) override { + if (FLAGS_read_fault_one_in) { + (void)fault_fs_guard->GetAndResetErrorCount(); + fault_fs_guard->DisableErrorInjection(); + } + } + void OnTableFileCreationStarted( const TableFileCreationBriefInfo& /*info*/) override { ++num_pending_file_creations_; diff --git a/db_stress_tool/db_stress_shared_state.h b/db_stress_tool/db_stress_shared_state.h index 0137f0b2e..bad6a77e1 100644 --- a/db_stress_tool/db_stress_shared_state.h +++ b/db_stress_tool/db_stress_shared_state.h @@ -35,7 +35,7 @@ DECLARE_int32(open_metadata_write_fault_one_in); DECLARE_int32(open_write_fault_one_in); DECLARE_int32(open_read_fault_one_in); -DECLARE_int32(injest_error_severity); +DECLARE_int32(inject_error_severity); namespace ROCKSDB_NAMESPACE { class StressTest; @@ -342,6 +342,13 @@ class SharedState { uint64_t GetStartTimestamp() const { return start_timestamp_; } + void SafeTerminate() { + // Grab mutex so that we don't call terminate while another thread is + // attempting to print a stack trace due to the first one + MutexLock l(&mu_); + std::terminate(); + } + private: static void IgnoreReadErrorCallback(void*) { ignore_read_error = true; } diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 2c62049c3..e79a71127 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -11,12 +11,15 @@ #include #include +#include "rocksdb/options.h" #include "util/compression.h" #ifdef GFLAGS #include "db_stress_tool/db_stress_common.h" #include "db_stress_tool/db_stress_compaction_filter.h" #include "db_stress_tool/db_stress_driver.h" #include "db_stress_tool/db_stress_table_properties_collector.h" +#include "db_stress_tool/db_stress_wide_merge_operator.h" +#include "options/options_parser.h" #include "rocksdb/convenience.h" #include "rocksdb/filter_policy.h" #include "rocksdb/secondary_cache.h" @@ -39,12 +42,12 @@ std::shared_ptr CreateFilterPolicy() { return BlockBasedTableOptions().filter_policy; } const FilterPolicy* new_policy; - if (FLAGS_ribbon_starting_level >= 999) { + if (FLAGS_bloom_before_level == INT_MAX) { // Use Bloom API new_policy = NewBloomFilterPolicy(FLAGS_bloom_bits, false); } else { - new_policy = NewRibbonFilterPolicy( - FLAGS_bloom_bits, /* bloom_before_level */ FLAGS_ribbon_starting_level); + new_policy = + NewRibbonFilterPolicy(FLAGS_bloom_bits, FLAGS_bloom_before_level); } return std::shared_ptr(new_policy); } @@ -111,6 +114,11 @@ std::shared_ptr StressTest::NewCache(size_t capacity, std::shared_ptr secondary_cache; if (!FLAGS_secondary_cache_uri.empty()) { + assert(!strstr(FLAGS_secondary_cache_uri.c_str(), + "compressed_secondary_cache") || + (FLAGS_compressed_secondary_cache_size == 0 && + FLAGS_compressed_secondary_cache_ratio == 0.0 && + !StartsWith(FLAGS_cache_type, "tiered_"))); Status s = SecondaryCache::CreateFromString( config_options, FLAGS_secondary_cache_uri, &secondary_cache); if (secondary_cache == nullptr) { @@ -124,27 +132,81 @@ std::shared_ptr StressTest::NewCache(size_t capacity, secondary_cache, static_cast(FLAGS_seed), FLAGS_secondary_cache_fault_one_in); } + } else if (FLAGS_compressed_secondary_cache_size > 0) { + if (StartsWith(FLAGS_cache_type, "tiered_")) { + fprintf(stderr, + "Cannot specify both compressed_secondary_cache_size and %s\n", + FLAGS_cache_type.c_str()); + exit(1); + } + CompressedSecondaryCacheOptions opts; + opts.capacity = FLAGS_compressed_secondary_cache_size; + secondary_cache = NewCompressedSecondaryCache(opts); + if (secondary_cache == nullptr) { + fprintf(stderr, "Failed to allocate compressed secondary cache\n"); + exit(1); + } + compressed_secondary_cache = secondary_cache; } - if (FLAGS_cache_type == "clock_cache") { + std::string cache_type = FLAGS_cache_type; + size_t cache_size = FLAGS_cache_size; + bool tiered = false; + if (StartsWith(cache_type, "tiered_")) { + tiered = true; + cache_type.erase(0, strlen("tiered_")); + } + if (FLAGS_use_write_buffer_manager) { + cache_size += FLAGS_db_write_buffer_size; + } + if (cache_type == "clock_cache") { fprintf(stderr, "Old clock cache implementation has been removed.\n"); exit(1); - } else if (FLAGS_cache_type == "hyper_clock_cache") { - HyperClockCacheOptions opts(static_cast(capacity), - FLAGS_block_size /*estimated_entry_charge*/, + } else if (EndsWith(cache_type, "hyper_clock_cache")) { + size_t estimated_entry_charge; + if (cache_type == "fixed_hyper_clock_cache" || + cache_type == "hyper_clock_cache") { + estimated_entry_charge = FLAGS_block_size; + } else if (cache_type == "auto_hyper_clock_cache") { + estimated_entry_charge = 0; + } else { + fprintf(stderr, "Cache type not supported."); + exit(1); + } + HyperClockCacheOptions opts(cache_size, estimated_entry_charge, num_shard_bits); - opts.secondary_cache = std::move(secondary_cache); - return opts.MakeSharedCache(); - } else if (FLAGS_cache_type == "lru_cache") { + opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX); + if (tiered) { + TieredCacheOptions tiered_opts; + tiered_opts.cache_opts = &opts; + tiered_opts.cache_type = PrimaryCacheType::kCacheTypeHCC; + tiered_opts.total_capacity = cache_size; + tiered_opts.compressed_secondary_ratio = 0.5; + block_cache = NewTieredCache(tiered_opts); + } else { + opts.secondary_cache = std::move(secondary_cache); + block_cache = opts.MakeSharedCache(); + } + } else if (EndsWith(cache_type, "lru_cache")) { LRUCacheOptions opts; opts.capacity = capacity; opts.num_shard_bits = num_shard_bits; - opts.secondary_cache = std::move(secondary_cache); - return NewLRUCache(opts); + if (tiered) { + TieredCacheOptions tiered_opts; + tiered_opts.cache_opts = &opts; + tiered_opts.cache_type = PrimaryCacheType::kCacheTypeLRU; + tiered_opts.total_capacity = cache_size; + tiered_opts.compressed_secondary_ratio = 0.5; + block_cache = NewTieredCache(tiered_opts); + } else { + opts.secondary_cache = std::move(secondary_cache); + block_cache = NewLRUCache(opts); + } } else { fprintf(stderr, "Cache type not supported."); exit(1); } + return block_cache; } std::vector StressTest::GetBlobCompressionTags() { @@ -264,6 +326,13 @@ bool StressTest::BuildOptionsTable() { std::vector{"kDisable", "kFlushOnly"}); } + if (FLAGS_bloom_before_level != INT_MAX) { + // Can modify RibbonFilterPolicy field + options_tbl.emplace("table_factory.filter_policy.bloom_before_level", + std::vector{"-1", "0", "1", "2", + "2147483646", "2147483647"}); + } + options_table_ = std::move(options_tbl); for (const auto& iter : options_table_) { @@ -361,6 +430,13 @@ Status StressTest::AssertSame(DB* db, ColumnFamilyHandle* cf, PinnableSlice v; s = db->Get(ropt, cf, snap_state.key, &v); if (!s.ok() && !s.IsNotFound()) { + // When `persist_user_defined_timestamps` is false, a repeated read with + // both a read timestamp and an explicitly taken snapshot cannot guarantee + // consistent result all the time. When it cannot return consistent result, + // it will return an `InvalidArgument` status. + if (s.IsInvalidArgument() && !FLAGS_persist_user_defined_timestamps) { + return Status::OK(); + } return s; } if (snap_state.status != s) { @@ -398,10 +474,22 @@ Status StressTest::AssertSame(DB* db, ColumnFamilyHandle* cf, return Status::OK(); } -void StressTest::VerificationAbort(SharedState* shared, std::string msg, - Status s) const { - fprintf(stderr, "Verification failed: %s. Status is %s\n", msg.c_str(), - s.ToString().c_str()); +void StressTest::ProcessStatus(SharedState* shared, std::string opname, + Status s) const { + if (s.ok()) { + return; + } + if (!s.IsIOError() || !std::strstr(s.getState(), "injected")) { + std::ostringstream oss; + oss << opname << " failed: " << s.ToString(); + VerificationAbort(shared, oss.str()); + assert(false); + } + fprintf(stdout, "%s failed: %s\n", opname.c_str(), s.ToString().c_str()); +} + +void StressTest::VerificationAbort(SharedState* shared, std::string msg) const { + fprintf(stderr, "Verification failed: %s\n", msg.c_str()); shared->SetVerificationFailure(); } @@ -490,13 +578,16 @@ void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys, const Slice v(value, sz); - std::string ts; if (FLAGS_user_timestamp_size > 0) { ts = GetNowNanos(); } - if (FLAGS_use_merge) { + if (FLAGS_use_put_entity_one_in > 0 && + (value_base % FLAGS_use_put_entity_one_in) == 0) { + s = db_->PutEntity(write_opts, cfh, key, + GenerateWideColumns(value_base, v)); + } else if (FLAGS_use_merge) { if (!FLAGS_use_txn) { if (FLAGS_user_timestamp_size > 0) { s = db_->Merge(write_opts, cfh, key, ts, v); @@ -504,18 +595,10 @@ void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys, s = db_->Merge(write_opts, cfh, key, v); } } else { - Transaction* txn; - s = NewTxn(write_opts, &txn); - if (s.ok()) { - s = txn->Merge(cfh, key, v); - if (s.ok()) { - s = CommitTxn(txn); - } - } + s = ExecuteTransaction( + write_opts, /*thread=*/nullptr, + [&](Transaction& txn) { return txn.Merge(cfh, key, v); }); } - } else if (FLAGS_use_put_entity_one_in > 0) { - s = db_->PutEntity(write_opts, cfh, key, - GenerateWideColumns(value_base, v)); } else { if (!FLAGS_use_txn) { if (FLAGS_user_timestamp_size > 0) { @@ -524,14 +607,9 @@ void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys, s = db_->Put(write_opts, cfh, key, v); } } else { - Transaction* txn; - s = NewTxn(write_opts, &txn); - if (s.ok()) { - s = txn->Put(cfh, key, v); - if (s.ok()) { - s = CommitTxn(txn); - } - } + s = ExecuteTransaction( + write_opts, /*thread=*/nullptr, + [&](Transaction& txn) { return txn.Put(cfh, key, v); }); } } @@ -629,14 +707,15 @@ void StressTest::ProcessRecoveredPreparedTxnsHelper(Transaction* txn, } } -Status StressTest::NewTxn(WriteOptions& write_opts, Transaction** txn) { +Status StressTest::NewTxn(WriteOptions& write_opts, + std::unique_ptr* out_txn) { if (!FLAGS_use_txn) { return Status::InvalidArgument("NewTxn when FLAGS_use_txn is not set"); } write_opts.disableWAL = FLAGS_disable_wal; static std::atomic txn_id = {0}; if (FLAGS_use_optimistic_txn) { - *txn = optimistic_txn_db_->BeginTransaction(write_opts); + out_txn->reset(optimistic_txn_db_->BeginTransaction(write_opts)); return Status::OK(); } else { TransactionOptions txn_options; @@ -644,31 +723,31 @@ Status StressTest::NewTxn(WriteOptions& write_opts, Transaction** txn) { FLAGS_use_only_the_last_commit_time_batch_for_recovery; txn_options.lock_timeout = 600000; // 10 min txn_options.deadlock_detect = true; - *txn = txn_db_->BeginTransaction(write_opts, txn_options); + out_txn->reset(txn_db_->BeginTransaction(write_opts, txn_options)); auto istr = std::to_string(txn_id.fetch_add(1)); - Status s = (*txn)->SetName("xid" + istr); + Status s = (*out_txn)->SetName("xid" + istr); return s; } } -Status StressTest::CommitTxn(Transaction* txn, ThreadState* thread) { +Status StressTest::CommitTxn(Transaction& txn, ThreadState* thread) { if (!FLAGS_use_txn) { return Status::InvalidArgument("CommitTxn when FLAGS_use_txn is not set"); } Status s = Status::OK(); if (FLAGS_use_optimistic_txn) { assert(optimistic_txn_db_); - s = txn->Commit(); + s = txn.Commit(); } else { assert(txn_db_); - s = txn->Prepare(); + s = txn.Prepare(); std::shared_ptr timestamped_snapshot; if (s.ok()) { if (thread && FLAGS_create_timestamped_snapshot_one_in && thread->rand.OneIn(FLAGS_create_timestamped_snapshot_one_in)) { uint64_t ts = db_stress_env->NowNanos(); - s = txn->CommitAndTryCreateSnapshot(/*notifier=*/nullptr, ts, - ×tamped_snapshot); + s = txn.CommitAndTryCreateSnapshot(/*notifier=*/nullptr, ts, + ×tamped_snapshot); std::pair> res; if (thread->tid == 0) { @@ -686,7 +765,7 @@ Status StressTest::CommitTxn(Transaction* txn, ThreadState* thread) { } } } else { - s = txn->Commit(); + s = txn.Commit(); } } if (thread && FLAGS_create_timestamped_snapshot_one_in > 0 && @@ -696,18 +775,48 @@ Status StressTest::CommitTxn(Transaction* txn, ThreadState* thread) { txn_db_->ReleaseTimestampedSnapshotsOlderThan(now - time_diff); } } - delete txn; return s; } -Status StressTest::RollbackTxn(Transaction* txn) { - if (!FLAGS_use_txn) { - return Status::InvalidArgument( - "RollbackTxn when FLAGS_use_txn is not" - " set"); +Status StressTest::ExecuteTransaction( + WriteOptions& write_opts, ThreadState* thread, + std::function&& ops) { + std::unique_ptr txn; + Status s = NewTxn(write_opts, &txn); + std::string try_again_messages; + if (s.ok()) { + for (int tries = 1;; ++tries) { + s = ops(*txn); + if (s.ok()) { + s = CommitTxn(*txn, thread); + if (s.ok()) { + break; + } + } + // Optimistic txn might return TryAgain, in which case rollback + // and try again. + if (!s.IsTryAgain() || !FLAGS_use_optimistic_txn) { + break; + } + // Record and report historical TryAgain messages for debugging + try_again_messages += + std::to_string(SystemClock::Default()->NowMicros() / 1000); + try_again_messages += "ms "; + try_again_messages += s.getState(); + try_again_messages += "\n"; + // In theory, each Rollback after TryAgain should have an independent + // chance of success, so too many retries could indicate something is + // not working properly. + if (tries >= 10) { + s = Status::TryAgain(try_again_messages); + break; + } + s = txn->Rollback(); + if (!s.ok()) { + break; + } + } } - Status s = txn->Rollback(); - delete txn; return s; } @@ -718,6 +827,7 @@ void StressTest::OperateDb(ThreadState* thread) { read_opts.async_io = FLAGS_async_io; read_opts.adaptive_readahead = FLAGS_adaptive_readahead; read_opts.readahead_size = FLAGS_readahead_size; + read_opts.auto_readahead_size = FLAGS_auto_readahead_size; WriteOptions write_opts; if (FLAGS_rate_limit_auto_wal_flush) { write_opts.rate_limiter_priority = Env::IO_USER; @@ -743,27 +853,11 @@ void StressTest::OperateDb(ThreadState* thread) { #ifndef NDEBUG if (FLAGS_read_fault_one_in) { - fault_fs_guard->SetThreadLocalReadErrorContext(thread->shared->GetSeed(), - FLAGS_read_fault_one_in); + fault_fs_guard->SetThreadLocalReadErrorContext( + thread->shared->GetSeed(), FLAGS_read_fault_one_in, + FLAGS_inject_error_severity == 1 /* retryable */); } #endif // NDEBUG - if (FLAGS_write_fault_one_in) { - IOStatus error_msg; - if (FLAGS_injest_error_severity <= 1 || FLAGS_injest_error_severity > 2) { - error_msg = IOStatus::IOError("Retryable IO Error"); - error_msg.SetRetryable(true); - } else if (FLAGS_injest_error_severity == 2) { - // Ingest the fatal error - error_msg = IOStatus::IOError("Fatal IO Error"); - error_msg.SetDataLoss(true); - } - std::vector types = {FileType::kTableFile, - FileType::kDescriptorFile, - FileType::kCurrentFile}; - fault_fs_guard->SetRandomWriteError( - thread->shared->GetSeed(), FLAGS_write_fault_one_in, error_msg, - /*inject_for_all_file_types=*/false, types); - } thread->stats.Start(); for (int open_cnt = 0; open_cnt <= FLAGS_reopen; ++open_cnt) { if (thread->shared->HasVerificationFailedYet() || @@ -887,42 +981,42 @@ void StressTest::OperateDb(ThreadState* thread) { if (thread->rand.OneInOpt(FLAGS_get_live_files_one_in) && !FLAGS_write_fault_one_in) { Status status = VerifyGetLiveFiles(); - if (!status.ok()) { - VerificationAbort(shared, "VerifyGetLiveFiles status not OK", status); - } + ProcessStatus(shared, "VerifyGetLiveFiles", status); } // Verify GetSortedWalFiles with a 1 in N chance. if (thread->rand.OneInOpt(FLAGS_get_sorted_wal_files_one_in)) { Status status = VerifyGetSortedWalFiles(); - if (!status.ok()) { - VerificationAbort(shared, "VerifyGetSortedWalFiles status not OK", - status); - } + ProcessStatus(shared, "VerifyGetSortedWalFiles", status); } // Verify GetCurrentWalFile with a 1 in N chance. if (thread->rand.OneInOpt(FLAGS_get_current_wal_file_one_in)) { Status status = VerifyGetCurrentWalFile(); - if (!status.ok()) { - VerificationAbort(shared, "VerifyGetCurrentWalFile status not OK", - status); - } + ProcessStatus(shared, "VerifyGetCurrentWalFile", status); } if (thread->rand.OneInOpt(FLAGS_pause_background_one_in)) { Status status = TestPauseBackground(thread); - if (!status.ok()) { - VerificationAbort( - shared, "Pause/ContinueBackgroundWork status not OK", status); - } + ProcessStatus(shared, "Pause/ContinueBackgroundWork", status); } if (thread->rand.OneInOpt(FLAGS_verify_checksum_one_in)) { + ThreadStatusUtil::SetEnableTracking(FLAGS_enable_thread_tracking); + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_VERIFY_DB_CHECKSUM); Status status = db_->VerifyChecksum(); - if (!status.ok()) { - VerificationAbort(shared, "VerifyChecksum status not OK", status); - } + ThreadStatusUtil::ResetThreadStatus(); + ProcessStatus(shared, "VerifyChecksum", status); + } + + if (thread->rand.OneInOpt(FLAGS_verify_file_checksums_one_in)) { + ThreadStatusUtil::SetEnableTracking(FLAGS_enable_thread_tracking); + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_VERIFY_FILE_CHECKSUMS); + Status status = db_->VerifyFileChecksums(read_opts); + ThreadStatusUtil::ResetThreadStatus(); + ProcessStatus(shared, "VerifyFileChecksums", status); } if (thread->rand.OneInOpt(FLAGS_get_property_one_in)) { @@ -949,26 +1043,19 @@ void StressTest::OperateDb(ThreadState* thread) { if (total_size <= FLAGS_backup_max_size) { Status s = TestBackupRestore(thread, rand_column_families, rand_keys); - if (!s.ok()) { - VerificationAbort(shared, "Backup/restore gave inconsistent state", - s); - } + ProcessStatus(shared, "Backup/restore", s); } } if (thread->rand.OneInOpt(FLAGS_checkpoint_one_in)) { Status s = TestCheckpoint(thread, rand_column_families, rand_keys); - if (!s.ok()) { - VerificationAbort(shared, "Checkpoint gave inconsistent state", s); - } + ProcessStatus(shared, "Checkpoint", s); } if (thread->rand.OneInOpt(FLAGS_approximate_size_one_in)) { Status s = TestApproximateSize(thread, i, rand_column_families, rand_keys); - if (!s.ok()) { - VerificationAbort(shared, "ApproximateSize Failed", s); - } + ProcessStatus(shared, "ApproximateSize", s); } if (thread->rand.OneInOpt(FLAGS_acquire_snapshot_one_in)) { TestAcquireSnapshot(thread, rand_column_family, keystr, i); @@ -976,9 +1063,7 @@ void StressTest::OperateDb(ThreadState* thread) { /*always*/ { Status s = MaybeReleaseSnapshots(thread, i); - if (!s.ok()) { - VerificationAbort(shared, "Snapshot gave inconsistent state", s); - } + ProcessStatus(shared, "Snapshot", s); } // Assign timestamps if necessary. @@ -997,6 +1082,7 @@ void StressTest::OperateDb(ThreadState* thread) { if (prob_op >= 0 && prob_op < static_cast(FLAGS_readpercent)) { assert(0 <= prob_op); // OPERATION read + ThreadStatusUtil::SetEnableTracking(FLAGS_enable_thread_tracking); if (FLAGS_use_multi_get_entity) { constexpr uint64_t max_batch_size = 64; const uint64_t batch_size = std::min( @@ -1007,12 +1093,14 @@ void StressTest::OperateDb(ThreadState* thread) { assert(i + batch_size <= ops_per_open); rand_keys = GenerateNKeys(thread, static_cast(batch_size), i); - + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_MULTIGETENTITY); TestMultiGetEntity(thread, read_opts, rand_column_families, rand_keys); - i += batch_size - 1; } else if (FLAGS_use_get_entity) { + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_GETENTITY); TestGetEntity(thread, read_opts, rand_column_families, rand_keys); } else if (FLAGS_use_multiget) { // Leave room for one more iteration of the loop with a single key @@ -1024,11 +1112,16 @@ void StressTest::OperateDb(ThreadState* thread) { // If its the last iteration, ensure that multiget_batch_size is 1 multiget_batch_size = std::max(multiget_batch_size, 1); rand_keys = GenerateNKeys(thread, multiget_batch_size, i); + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_MULTIGET); TestMultiGet(thread, read_opts, rand_column_families, rand_keys); i += multiget_batch_size - 1; } else { + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_GET); TestGet(thread, read_opts, rand_column_families, rand_keys); } + ThreadStatusUtil::ResetThreadStatus(); } else if (prob_op < prefix_bound) { assert(static_cast(FLAGS_readpercent) <= prob_op); // OPERATION prefix scan @@ -1056,8 +1149,12 @@ void StressTest::OperateDb(ThreadState* thread) { if (!FLAGS_skip_verifydb && thread->rand.OneInOpt( FLAGS_verify_iterator_with_expected_state_one_in)) { + ThreadStatusUtil::SetEnableTracking(FLAGS_enable_thread_tracking); + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_DBITERATOR); TestIterateAgainstExpected(thread, read_opts, rand_column_families, rand_keys); + ThreadStatusUtil::ResetThreadStatus(); } else { int num_seeks = static_cast(std::min( std::max(static_cast(thread->rand.Uniform(4)), @@ -1066,7 +1163,11 @@ void StressTest::OperateDb(ThreadState* thread) { static_cast(1)))); rand_keys = GenerateNKeys(thread, num_seeks, i); i += num_seeks - 1; + ThreadStatusUtil::SetEnableTracking(FLAGS_enable_thread_tracking); + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_DBITERATOR); TestIterate(thread, read_opts, rand_column_families, rand_keys); + ThreadStatusUtil::ResetThreadStatus(); } } else { assert(iterate_bound <= prob_op); @@ -1173,7 +1274,6 @@ Status StressTest::TestIterate(ThreadState* thread, } else if (options_.prefix_extractor.get() == nullptr) { expect_total_order = true; } - std::string upper_bound_str; Slice upper_bound; if (thread->rand.OneIn(16)) { @@ -1184,6 +1284,7 @@ Status StressTest::TestIterate(ThreadState* thread, upper_bound = Slice(upper_bound_str); ro.iterate_upper_bound = &upper_bound; } + std::string lower_bound_str; Slice lower_bound; if (thread->rand.OneIn(16)) { @@ -1278,6 +1379,14 @@ Status StressTest::TestIterate(ThreadState* thread, const bool support_seek_first_or_last = expect_total_order; + // Write-prepared and Write-unprepared do not support Refresh() yet. + if (!(FLAGS_use_txn && FLAGS_txn_write_policy != 0 /* write committed */) && + thread->rand.OneIn(4)) { + Status s = iter->Refresh(snapshot_guard.snapshot()); + assert(s.ok()); + op_logs += "Refresh "; + } + LastIterateOp last_op; if (support_seek_first_or_last && thread->rand.OneIn(100)) { iter->SeekToFirst(); @@ -1383,11 +1492,11 @@ void StressTest::VerifyIterator(ThreadState* thread, } if (op == kLastOpSeekToFirst && ro.iterate_lower_bound != nullptr) { - // SeekToFirst() with lower bound is not well defined. + // SeekToFirst() with lower bound is not well-defined. *diverged = true; return; } else if (op == kLastOpSeekToLast && ro.iterate_upper_bound != nullptr) { - // SeekToLast() with higher bound is not well defined. + // SeekToLast() with higher bound is not well-defined. *diverged = true; return; } else if (op == kLastOpSeek && ro.iterate_lower_bound != nullptr && @@ -1398,7 +1507,7 @@ void StressTest::VerifyIterator(ThreadState* thread, options_.comparator->CompareWithoutTimestamp( *ro.iterate_lower_bound, /*a_has_ts=*/false, *ro.iterate_upper_bound, /*b_has_ts*/ false) >= 0))) { - // Lower bound behavior is not well defined if it is larger than + // Lower bound behavior is not well-defined if it is larger than // seek key or upper bound. Disable the check for now. *diverged = true; return; @@ -1410,7 +1519,7 @@ void StressTest::VerifyIterator(ThreadState* thread, options_.comparator->CompareWithoutTimestamp( *ro.iterate_lower_bound, /*a_has_ts=*/false, *ro.iterate_upper_bound, /*b_has_ts=*/false) >= 0))) { - // Uppder bound behavior is not well defined if it is smaller than + // Upper bound behavior is not well-defined if it is smaller than // seek key or lower bound. Disable the check for now. *diverged = true; return; @@ -1438,13 +1547,13 @@ void StressTest::VerifyIterator(ThreadState* thread, } } fprintf(stderr, - "Control interator is invalid but iterator has key %s " + "Control iterator is invalid but iterator has key %s " "%s\n", iter->key().ToString(true).c_str(), op_logs.c_str()); *diverged = true; } else if (cmp_iter->Valid()) { - // Iterator is not valid. It can be legimate if it has already been + // Iterator is not valid. It can be legitimate if it has already been // out of upper or lower bound, or filtered out by prefix iterator. const Slice& total_order_key = cmp_iter->key(); @@ -1467,7 +1576,7 @@ void StressTest::VerifyIterator(ThreadState* thread, return; } fprintf(stderr, - "Iterator stays in prefix but contol doesn't" + "Iterator stays in prefix but control doesn't" " iterator key %s control iterator key %s %s\n", iter->key().ToString(true).c_str(), cmp_iter->key().ToString(true).c_str(), op_logs.c_str()); @@ -1493,7 +1602,8 @@ void StressTest::VerifyIterator(ThreadState* thread, fprintf(stderr, "iterator has value %s\n", iter->key().ToString(true).c_str()); } else { - fprintf(stderr, "iterator is not valid\n"); + fprintf(stderr, "iterator is not valid with status: %s\n", + iter->status().ToString().c_str()); } *diverged = true; } @@ -1513,7 +1623,8 @@ void StressTest::VerifyIterator(ThreadState* thread, } if (*diverged) { - fprintf(stderr, "Control CF %s\n", cmp_cfh->GetName().c_str()); + fprintf(stderr, "VerifyIterator failed. Control CF %s\n", + cmp_cfh->GetName().c_str()); thread->stats.AddErrors(1); // Fail fast to preserve the DB state. thread->shared->SetVerificationFailure(); @@ -1664,20 +1775,23 @@ Status StressTest::TestBackupRestore( } DB* restored_db = nullptr; std::vector restored_cf_handles; + // Not yet implemented: opening restored BlobDB or TransactionDB + Options restore_options; + if (s.ok() && !FLAGS_use_txn && !FLAGS_use_blob_db) { + s = PrepareOptionsForRestoredDB(&restore_options); + if (!s.ok()) { + from = "PrepareRestoredDBOptions in backup/restore"; + } + } if (s.ok() && !FLAGS_use_txn && !FLAGS_use_blob_db) { - Options restore_options(options_); - restore_options.best_efforts_recovery = false; - restore_options.listeners.clear(); - // Avoid dangling/shared file descriptors, for reliable destroy - restore_options.sst_file_manager = nullptr; std::vector cf_descriptors; // TODO(ajkr): `column_family_names_` is not safe to access here when // `clear_column_family_one_in != 0`. But we can't easily switch to // `ListColumnFamilies` to get names because it won't necessarily give // the same order as `column_family_names_`. assert(FLAGS_clear_column_family_one_in == 0); - for (auto name : column_family_names_) { + for (const auto& name : column_family_names_) { cf_descriptors.emplace_back(name, ColumnFamilyOptions(restore_options)); } if (inplace_not_restore) { @@ -1781,13 +1895,75 @@ Status StressTest::TestBackupRestore( from = "Destroy restore dir"; } } - if (!s.ok()) { + if (!s.ok() && (!s.IsIOError() || !std::strstr(s.getState(), "injected"))) { fprintf(stderr, "Failure in %s with: %s\n", from.c_str(), s.ToString().c_str()); } return s; } +void InitializeMergeOperator(Options& options) { + if (FLAGS_use_full_merge_v1) { + options.merge_operator = MergeOperators::CreateDeprecatedPutOperator(); + } else { + if (FLAGS_use_put_entity_one_in > 0) { + options.merge_operator = std::make_shared(); + } else { + options.merge_operator = MergeOperators::CreatePutOperator(); + } + } +} + +Status StressTest::PrepareOptionsForRestoredDB(Options* options) { + assert(options); + // To avoid race with other threads' operations (e.g, SetOptions()) + // on the same pointer sub-option (e.g, `std::shared_ptr + // filter_policy`) while having the same settings as `options_`, we create a + // new Options object from `options_`'s string to deep copy these pointer + // sub-options + Status s; + ConfigOptions config_opts; + + std::string db_options_str; + s = GetStringFromDBOptions(config_opts, options_, &db_options_str); + if (!s.ok()) { + return s; + } + DBOptions db_options; + s = GetDBOptionsFromString(config_opts, Options(), db_options_str, + &db_options); + if (!s.ok()) { + return s; + } + + std::string cf_options_str; + s = GetStringFromColumnFamilyOptions(config_opts, options_, &cf_options_str); + if (!s.ok()) { + return s; + } + ColumnFamilyOptions cf_options; + s = GetColumnFamilyOptionsFromString(config_opts, Options(), cf_options_str, + &cf_options); + if (!s.ok()) { + return s; + } + + *options = Options(db_options, cf_options); + options->best_efforts_recovery = false; + options->listeners.clear(); + // Avoid dangling/shared file descriptors, for reliable destroy + options->sst_file_manager = nullptr; + // GetColumnFamilyOptionsFromString does not create customized merge operator. + InitializeMergeOperator(*options); + if (FLAGS_user_timestamp_size > 0) { + // Check OPTIONS string loading can bootstrap the correct user comparator + // from object registry. + assert(options->comparator); + assert(options->comparator == test::BytewiseComparatorWithU64TsWrapper()); + } + + return Status::OK(); +} Status StressTest::TestApproximateSize( ThreadState* thread, uint64_t iteration, const std::vector& rand_column_families, @@ -1857,17 +2033,19 @@ Status StressTest::TestCheckpoint(ThreadState* thread, if (s.ok()) { s = checkpoint->CreateCheckpoint(checkpoint_dir); if (!s.ok()) { - fprintf(stderr, "Fail to create checkpoint to %s\n", - checkpoint_dir.c_str()); - std::vector files; - Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files); - if (my_s.ok()) { - for (const auto& f : files) { - fprintf(stderr, " %s\n", f.c_str()); + if (!s.IsIOError() || !std::strstr(s.getState(), "injected")) { + fprintf(stderr, "Fail to create checkpoint to %s\n", + checkpoint_dir.c_str()); + std::vector files; + Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files); + if (my_s.ok()) { + for (const auto& f : files) { + fprintf(stderr, " %s\n", f.c_str()); + } + } else { + fprintf(stderr, "Fail to get files under the directory to %s\n", + my_s.ToString().c_str()); } - } else { - fprintf(stderr, "Fail to get files under the directory to %s\n", - my_s.ToString().c_str()); } } } @@ -1943,8 +2121,10 @@ Status StressTest::TestCheckpoint(ThreadState* thread, } if (!s.ok()) { - fprintf(stderr, "A checkpoint operation failed with: %s\n", - s.ToString().c_str()); + if (!s.IsIOError() || !std::strstr(s.getState(), "injected")) { + fprintf(stderr, "A checkpoint operation failed with: %s\n", + s.ToString().c_str()); + } } else { DestroyDB(checkpoint_dir, tmp_opts); } @@ -2423,6 +2603,8 @@ void StressTest::PrintEnv() const { FLAGS_use_get_entity ? "true" : "false"); fprintf(stdout, "Use MultiGetEntity : %s\n", FLAGS_use_multi_get_entity ? "true" : "false"); + fprintf(stdout, "Verification only : %s\n", + FLAGS_verification_only ? "true" : "false"); const char* memtablerep = ""; switch (FLAGS_rep_factory) { @@ -2493,6 +2675,8 @@ void StressTest::PrintEnv() const { static_cast(FLAGS_fail_if_options_file_error)); fprintf(stdout, "User timestamp size bytes : %d\n", static_cast(FLAGS_user_timestamp_size)); + fprintf(stdout, "Persist user defined timestamps : %d\n", + FLAGS_persist_user_defined_timestamps); fprintf(stdout, "WAL compression : %s\n", FLAGS_wal_compression.c_str()); fprintf(stdout, "Try verify sst unique id : %d\n", @@ -2501,7 +2685,7 @@ void StressTest::PrintEnv() const { fprintf(stdout, "------------------------------------------------\n"); } -void StressTest::Open(SharedState* shared) { +void StressTest::Open(SharedState* shared, bool reopen) { assert(db_ == nullptr); assert(txn_db_ == nullptr); assert(optimistic_txn_db_ == nullptr); @@ -2587,12 +2771,12 @@ void StressTest::Open(SharedState* shared) { if (sorted_cfn != existing_column_families) { fprintf(stderr, "Expected column families differ from the existing:\n"); fprintf(stderr, "Expected: {"); - for (auto cf : sorted_cfn) { + for (const auto& cf : sorted_cfn) { fprintf(stderr, "%s ", cf.c_str()); } fprintf(stderr, "}\n"); fprintf(stderr, "Existing: {"); - for (auto cf : existing_column_families) { + for (const auto& cf : existing_column_families) { fprintf(stderr, "%s ", cf.c_str()); } fprintf(stderr, "}\n"); @@ -2600,7 +2784,7 @@ void StressTest::Open(SharedState* shared) { assert(sorted_cfn == existing_column_families); } std::vector cf_descriptors; - for (auto name : column_family_names_) { + for (const auto& name : column_family_names_) { if (name != kDefaultColumnFamilyName) { new_column_family_name_ = std::max(new_column_family_name_.load(), std::stoi(name) + 1); @@ -2619,15 +2803,20 @@ void StressTest::Open(SharedState* shared) { FLAGS_db, options_.db_paths, cf_descriptors, db_stress_listener_env)); RegisterAdditionalListeners(); + // If this is for DB reopen, write error injection may have been enabled. + // Disable it here in case there is no open fault injection. + if (fault_fs_guard) { + fault_fs_guard->DisableWriteErrorInjection(); + } if (!FLAGS_use_txn) { - // Determine whether we need to ingest file metadata write failures + // Determine whether we need to inject file metadata write failures // during DB reopen. If it does, enable it. - // Only ingest metadata error if it is reopening, as initial open + // Only inject metadata error if it is reopening, as initial open // failure doesn't need to be handled. // TODO cover transaction DB is not covered in this fault test too. - bool ingest_meta_error = false; - bool ingest_write_error = false; - bool ingest_read_error = false; + bool inject_meta_error = false; + bool inject_write_error = false; + bool inject_read_error = false; if ((FLAGS_open_metadata_write_fault_one_in || FLAGS_open_write_fault_one_in || FLAGS_open_read_fault_one_in) && fault_fs_guard @@ -2638,25 +2827,25 @@ void StressTest::Open(SharedState* shared) { // WAL is durable. Buffering unsynced writes will cause false // positive in crash tests. Before we figure out a way to // solve it, skip WAL from failure injection. - fault_fs_guard->SetSkipDirectWritableTypes({kWalFile}); + fault_fs_guard->SetDirectWritableTypes({kWalFile}); } - ingest_meta_error = FLAGS_open_metadata_write_fault_one_in; - ingest_write_error = FLAGS_open_write_fault_one_in; - ingest_read_error = FLAGS_open_read_fault_one_in; - if (ingest_meta_error) { + inject_meta_error = FLAGS_open_metadata_write_fault_one_in; + inject_write_error = FLAGS_open_write_fault_one_in; + inject_read_error = FLAGS_open_read_fault_one_in; + if (inject_meta_error) { fault_fs_guard->EnableMetadataWriteErrorInjection(); fault_fs_guard->SetRandomMetadataWriteError( FLAGS_open_metadata_write_fault_one_in); } - if (ingest_write_error) { + if (inject_write_error) { fault_fs_guard->SetFilesystemDirectWritable(false); fault_fs_guard->EnableWriteErrorInjection(); fault_fs_guard->SetRandomWriteError( static_cast(FLAGS_seed), FLAGS_open_write_fault_one_in, - IOStatus::IOError("Injected Open Error"), + IOStatus::IOError("Injected Open Write Error"), /*inject_for_all_file_types=*/true, /*types=*/{}); } - if (ingest_read_error) { + if (inject_read_error) { fault_fs_guard->SetRandomReadError(FLAGS_open_read_fault_one_in); } } @@ -2677,8 +2866,7 @@ void StressTest::Open(SharedState* shared) { if (s.ok()) { db_ = blob_db; } - } else - { + } else { if (db_preload_finished_.load() && FLAGS_read_only) { s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db, cf_descriptors, &column_families_, &db_); @@ -2688,14 +2876,16 @@ void StressTest::Open(SharedState* shared) { } } - if (ingest_meta_error || ingest_write_error || ingest_read_error) { + if (inject_meta_error || inject_write_error || inject_read_error) { + // TODO: re-enable write error injection after reopen. Same for + // sync fault injection. fault_fs_guard->SetFilesystemDirectWritable(true); fault_fs_guard->DisableMetadataWriteErrorInjection(); fault_fs_guard->DisableWriteErrorInjection(); - fault_fs_guard->SetSkipDirectWritableTypes({}); + fault_fs_guard->SetDirectWritableTypes({}); fault_fs_guard->SetRandomReadError(0); if (s.ok()) { - // Ingested errors might happen in background compactions. We + // Injected errors might happen in background compactions. We // wait for all compactions to finish to make sure DB is in // clean state before executing queries. s = db_->GetRootDB()->WaitForCompact(WaitForCompactOptions()); @@ -2712,19 +2902,24 @@ void StressTest::Open(SharedState* shared) { // After failure to opening a DB due to IO error, retry should // successfully open the DB with correct data if no IO error shows // up. - ingest_meta_error = false; - ingest_write_error = false; - ingest_read_error = false; - - Random rand(static_cast(FLAGS_seed)); - if (rand.OneIn(2)) { - fault_fs_guard->DeleteFilesCreatedAfterLastDirSync(IOOptions(), - nullptr); - } - if (rand.OneIn(3)) { - fault_fs_guard->DropUnsyncedFileData(); - } else if (rand.OneIn(2)) { - fault_fs_guard->DropRandomUnsyncedFileData(&rand); + inject_meta_error = false; + inject_write_error = false; + inject_read_error = false; + + // TODO: Unsynced data loss during DB reopen is not supported yet in + // stress test. Will need to recreate expected state if we decide + // to support unsynced data loss during DB reopen. + if (!reopen) { + Random rand(static_cast(FLAGS_seed)); + if (rand.OneIn(2)) { + fault_fs_guard->DeleteFilesCreatedAfterLastDirSync(IOOptions(), + nullptr); + } + if (rand.OneIn(3)) { + fault_fs_guard->DropUnsyncedFileData(); + } else if (rand.OneIn(2)) { + fault_fs_guard->DropRandomUnsyncedFileData(&rand); + } } continue; } @@ -2879,7 +3074,7 @@ void StressTest::Reopen(ThreadState* thread) { auto now = clock_->NowMicros(); fprintf(stdout, "%s Reopening database for the %dth time\n", clock_->TimeToString(now / 1000000).c_str(), num_times_reopened_); - Open(thread->shared); + Open(thread->shared, /*reopen=*/true); if ((FLAGS_sync_fault_injection || FLAGS_disable_wal || FLAGS_manual_wal_flush_one_in > 0) && @@ -2901,6 +3096,11 @@ bool StressTest::MaybeUseOlderTimestampForPointLookup(ThreadState* thread, return false; } + if (!FLAGS_persist_user_defined_timestamps) { + // Not read with older timestamps to avoid get InvalidArgument. + return false; + } + assert(thread); if (!thread->rand.OneInOpt(3)) { return false; @@ -2930,6 +3130,11 @@ void StressTest::MaybeUseOlderTimestampForRangeScan(ThreadState* thread, return; } + if (!FLAGS_persist_user_defined_timestamps) { + // Not read with older timestamps to avoid get InvalidArgument. + return; + } + assert(thread); if (!thread->rand.OneInOpt(3)) { return; @@ -2989,6 +3194,8 @@ void CheckAndSetOptionsForUserTimestamp(Options& options) { exit(1); } options.comparator = cmp; + options.persist_user_defined_timestamps = + FLAGS_persist_user_defined_timestamps; } bool InitializeOptionsFromFile(Options& options) { @@ -3089,6 +3296,10 @@ void InitializeOptionsFromFlags( FLAGS_max_write_buffer_size_to_maintain; options.memtable_prefix_bloom_size_ratio = FLAGS_memtable_prefix_bloom_size_ratio; + if (FLAGS_use_write_buffer_manager) { + options.write_buffer_manager.reset( + new WriteBufferManager(FLAGS_db_write_buffer_size, block_cache)); + } options.memtable_whole_key_filtering = FLAGS_memtable_whole_key_filtering; options.disable_auto_compactions = FLAGS_disable_auto_compactions; options.max_background_compactions = FLAGS_max_background_compactions; @@ -3145,6 +3356,9 @@ void InitializeOptionsFromFlags( "cannot be used because ZSTD 1.4.5+ is not linked with the binary." " zstd dictionary trainer will be used.\n"); } + if (FLAGS_compression_checksum) { + options.compression_opts.checksum = true; + } options.max_manifest_file_size = FLAGS_max_manifest_file_size; options.inplace_update_support = FLAGS_in_place_update; options.max_subcompactions = static_cast(FLAGS_subcompactions); @@ -3246,11 +3460,8 @@ void InitializeOptionsFromFlags( options.memtable_factory.reset(new VectorRepFactory()); break; } - if (FLAGS_use_full_merge_v1) { - options.merge_operator = MergeOperators::CreateDeprecatedPutOperator(); - } else { - options.merge_operator = MergeOperators::CreatePutOperator(); - } + + InitializeMergeOperator(options); if (FLAGS_enable_compaction_filter) { options.compaction_filter_factory = @@ -3268,6 +3479,11 @@ void InitializeOptionsFromFlags( options.allow_data_in_errors = FLAGS_allow_data_in_errors; options.enable_thread_tracking = FLAGS_enable_thread_tracking; + + options.memtable_max_range_deletions = FLAGS_memtable_max_range_deletions; + + options.bottommost_file_compaction_delay = + FLAGS_bottommost_file_compaction_delay; } void InitializeOptionsGeneral( diff --git a/db_stress_tool/db_stress_test_base.h b/db_stress_tool/db_stress_test_base.h index 29159a494..424570b33 100644 --- a/db_stress_tool/db_stress_test_base.h +++ b/db_stress_tool/db_stress_test_base.h @@ -64,11 +64,14 @@ class StressTest { virtual void ProcessRecoveredPreparedTxnsHelper(Transaction* txn, SharedState* shared); - Status NewTxn(WriteOptions& write_opts, Transaction** txn); + // ExecuteTransaction is recommended instead + Status NewTxn(WriteOptions& write_opts, + std::unique_ptr* out_txn); + Status CommitTxn(Transaction& txn, ThreadState* thread = nullptr); - Status CommitTxn(Transaction* txn, ThreadState* thread = nullptr); - - Status RollbackTxn(Transaction* txn); + // Creates a transaction, executes `ops`, and tries to commit + Status ExecuteTransaction(WriteOptions& write_opts, ThreadState* thread, + std::function&& ops); virtual void MaybeClearOneColumnFamily(ThreadState* /* thread */) {} @@ -191,6 +194,8 @@ class StressTest { const std::vector& rand_column_families, const std::vector& rand_keys); + virtual Status PrepareOptionsForRestoredDB(Options* options); + virtual Status TestCheckpoint(ThreadState* thread, const std::vector& rand_column_families, const std::vector& rand_keys); @@ -221,7 +226,9 @@ class StressTest { return Status::NotSupported("TestCustomOperations() must be overridden"); } - void VerificationAbort(SharedState* shared, std::string msg, Status s) const; + void ProcessStatus(SharedState* shared, std::string msg, Status s) const; + + void VerificationAbort(SharedState* shared, std::string msg) const; void VerificationAbort(SharedState* shared, std::string msg, int cf, int64_t key) const; @@ -238,7 +245,7 @@ class StressTest { void PrintEnv() const; - void Open(SharedState* shared); + void Open(SharedState* shared, bool reopen = false); void Reopen(ThreadState* thread); diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc index f2b124b4c..9c24e2c42 100644 --- a/db_stress_tool/db_stress_tool.cc +++ b/db_stress_tool/db_stress_tool.cc @@ -88,11 +88,11 @@ int db_stress_tool(int argc, char** argv) { FaultInjectionTestFS* fs = new FaultInjectionTestFS(raw_env->GetFileSystem()); fault_fs_guard.reset(fs); - if (FLAGS_write_fault_one_in) { - fault_fs_guard->SetFilesystemDirectWritable(false); - } else { - fault_fs_guard->SetFilesystemDirectWritable(true); - } + // Set it to direct writable here to not lose files created during DB open + // when no open fault injection is not enabled. + // This will be overwritten in StressTest::Open() for open fault injection + // and in RunStressTestImpl() for proper write fault injection setup. + fault_fs_guard->SetFilesystemDirectWritable(true); fault_env_guard = std::make_shared(raw_env, fault_fs_guard); raw_env = fault_env_guard.get(); @@ -240,10 +240,10 @@ int db_stress_tool(int argc, char** argv) { FLAGS_secondaries_base = default_secondaries_path; } - if (FLAGS_best_efforts_recovery && !FLAGS_skip_verifydb && - !FLAGS_disable_wal) { + if (FLAGS_best_efforts_recovery && + !(FLAGS_skip_verifydb && FLAGS_disable_wal)) { fprintf(stderr, - "With best-efforts recovery, either skip_verifydb or disable_wal " + "With best-efforts recovery, skip_verifydb and disable_wal " "should be set to true.\n"); exit(1); } @@ -308,12 +308,11 @@ int db_stress_tool(int argc, char** argv) { } if (FLAGS_use_put_entity_one_in > 0 && - (FLAGS_ingest_external_file_one_in > 0 || FLAGS_use_merge || - FLAGS_use_full_merge_v1 || FLAGS_use_txn || FLAGS_test_multi_ops_txns || + (FLAGS_use_full_merge_v1 || FLAGS_use_txn || FLAGS_test_multi_ops_txns || FLAGS_user_timestamp_size > 0)) { fprintf(stderr, - "PutEntity is currently incompatible with SstFileWriter, Merge," - " transactions, and user-defined timestamps\n"); + "Wide columns are incompatible with V1 Merge, transactions, and " + "user-defined timestamps\n"); exit(1); } diff --git a/db_stress_tool/db_stress_wide_merge_operator.cc b/db_stress_tool/db_stress_wide_merge_operator.cc new file mode 100644 index 000000000..1fcfc3042 --- /dev/null +++ b/db_stress_tool/db_stress_wide_merge_operator.cc @@ -0,0 +1,51 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifdef GFLAGS + +#include "db_stress_tool/db_stress_wide_merge_operator.h" + +#include "db_stress_tool/db_stress_common.h" + +namespace ROCKSDB_NAMESPACE { + +bool DBStressWideMergeOperator::FullMergeV3( + const MergeOperationInputV3& merge_in, + MergeOperationOutputV3* merge_out) const { + assert(!merge_in.operand_list.empty()); + assert(merge_out); + + const Slice& latest = merge_in.operand_list.back(); + + if (latest.size() < sizeof(uint32_t)) { + return false; + } + + const uint32_t value_base = GetValueBase(latest); + + if (FLAGS_use_put_entity_one_in == 0 || + (value_base % FLAGS_use_put_entity_one_in) != 0) { + merge_out->new_value = latest; + return true; + } + + const auto columns = GenerateWideColumns(value_base, latest); + + merge_out->new_value = MergeOperationOutputV3::NewColumns(); + auto& new_columns = + std::get(merge_out->new_value); + new_columns.reserve(columns.size()); + + for (const auto& column : columns) { + new_columns.emplace_back(column.name().ToString(), + column.value().ToString()); + } + + return true; +} + +} // namespace ROCKSDB_NAMESPACE + +#endif // GFLAGS diff --git a/db_stress_tool/db_stress_wide_merge_operator.h b/db_stress_tool/db_stress_wide_merge_operator.h new file mode 100644 index 000000000..cba4f6b6b --- /dev/null +++ b/db_stress_tool/db_stress_wide_merge_operator.h @@ -0,0 +1,27 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/merge_operator.h" + +namespace ROCKSDB_NAMESPACE { + +// A test merge operator that implements the wide-column aware FullMergeV3 +// interface. Similarly to the simple "put" type merge operators, the merge +// result is based on the last merge operand; however, the merge result can +// potentially be a wide-column entity, depending on the value base encoded into +// the merge operand and the value of the "use_put_entity_one_in" stress test +// option. Following the same rule as for writes ensures that the queries +// issued by the validation logic receive the expected results. +class DBStressWideMergeOperator : public MergeOperator { + public: + bool FullMergeV3(const MergeOperationInputV3& merge_in, + MergeOperationOutputV3* merge_out) const override; + + const char* Name() const override { return "DBStressWideMergeOperator"; } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/db_stress_tool/expected_state.cc b/db_stress_tool/expected_state.cc index dd210cab4..705cd4f54 100644 --- a/db_stress_tool/expected_state.cc +++ b/db_stress_tool/expected_state.cc @@ -6,8 +6,8 @@ #include #ifdef GFLAGS - #include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "db_stress_tool/db_stress_common.h" #include "db_stress_tool/db_stress_shared_state.h" #include "db_stress_tool/expected_state.h" @@ -185,7 +185,7 @@ ExpectedStateManager::ExpectedStateManager(size_t max_key, num_column_families_(num_column_families), latest_(nullptr) {} -ExpectedStateManager::~ExpectedStateManager() {} +ExpectedStateManager::~ExpectedStateManager() = default; const std::string FileExpectedStateManager::kLatestBasename = "LATEST"; const std::string FileExpectedStateManager::kStateFilenameSuffix = ".state"; @@ -363,7 +363,6 @@ bool FileExpectedStateManager::HasHistory() { return saved_seqno_ != kMaxSequenceNumber; } - namespace { // An `ExpectedStateTraceRecordHandler` applies a configurable number of @@ -462,10 +461,8 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler, column_family_id, key, columns); } - assert(!columns.empty()); - assert(columns.front().name() == kDefaultWideColumnName); - - const uint32_t value_base = GetValueBase(columns.front().value()); + const uint32_t value_base = + GetValueBase(WideColumnsHelper::GetDefaultColumn(columns)); state_->SyncPut(column_family_id, static_cast(key_id), value_base); diff --git a/db_stress_tool/multi_ops_txns_stress.cc b/db_stress_tool/multi_ops_txns_stress.cc index 89b061004..145a96a75 100644 --- a/db_stress_tool/multi_ops_txns_stress.cc +++ b/db_stress_tool/multi_ops_txns_stress.cc @@ -150,7 +150,7 @@ std::string MultiOpsTxnsStressTest::Record::EncodePrimaryKey(uint32_t a) { PutFixed32(&ret, kPrimaryIndexId); PutFixed32(&ret, a); - char* const buf = &ret[0]; + char* const buf = ret.data(); std::reverse(buf, buf + sizeof(kPrimaryIndexId)); std::reverse(buf + sizeof(kPrimaryIndexId), buf + sizeof(kPrimaryIndexId) + sizeof(a)); @@ -162,7 +162,7 @@ std::string MultiOpsTxnsStressTest::Record::EncodeSecondaryKey(uint32_t c) { PutFixed32(&ret, kSecondaryIndexId); PutFixed32(&ret, c); - char* const buf = &ret[0]; + char* const buf = ret.data(); std::reverse(buf, buf + sizeof(kSecondaryIndexId)); std::reverse(buf + sizeof(kSecondaryIndexId), buf + sizeof(kSecondaryIndexId) + sizeof(c)); @@ -176,7 +176,7 @@ std::string MultiOpsTxnsStressTest::Record::EncodeSecondaryKey(uint32_t c, PutFixed32(&ret, c); PutFixed32(&ret, a); - char* const buf = &ret[0]; + char* const buf = ret.data(); std::reverse(buf, buf + sizeof(kSecondaryIndexId)); std::reverse(buf + sizeof(kSecondaryIndexId), buf + sizeof(kSecondaryIndexId) + sizeof(c)); @@ -560,7 +560,7 @@ Status MultiOpsTxnsStressTest::PrimaryKeyUpdateTxn(ThreadState* thread, uint32_t new_a) { std::string old_pk = Record::EncodePrimaryKey(old_a); std::string new_pk = Record::EncodePrimaryKey(new_a); - Transaction* txn = nullptr; + std::unique_ptr txn; WriteOptions wopts; Status s = NewTxn(wopts, &txn); if (!s.ok()) { @@ -572,7 +572,7 @@ Status MultiOpsTxnsStressTest::PrimaryKeyUpdateTxn(ThreadState* thread, assert(txn); txn->SetSnapshotOnNextOperation(/*notifier=*/nullptr); - const Defer cleanup([new_a, &s, thread, txn, this]() { + const Defer cleanup([new_a, &s, thread, this, &txn]() { if (s.ok()) { // Two gets, one for existing pk, one for locking potential new pk. thread->stats.AddGets(/*ngets=*/2, /*nfounds=*/1); @@ -594,7 +594,7 @@ Status MultiOpsTxnsStressTest::PrimaryKeyUpdateTxn(ThreadState* thread, } auto& key_gen = key_gen_for_a_[thread->tid]; key_gen->UndoAllocation(new_a); - RollbackTxn(txn).PermitUncheckedError(); + txn->Rollback().PermitUncheckedError(); }); ReadOptions ropts; @@ -671,7 +671,6 @@ Status MultiOpsTxnsStressTest::PrimaryKeyUpdateTxn(ThreadState* thread, auto& key_gen = key_gen_for_a_.at(thread->tid); if (s.ok()) { - delete txn; key_gen->Replace(old_a, old_a_pos, new_a); } return s; @@ -681,7 +680,7 @@ Status MultiOpsTxnsStressTest::SecondaryKeyUpdateTxn(ThreadState* thread, uint32_t old_c, uint32_t old_c_pos, uint32_t new_c) { - Transaction* txn = nullptr; + std::unique_ptr txn; WriteOptions wopts; Status s = NewTxn(wopts, &txn); if (!s.ok()) { @@ -694,7 +693,7 @@ Status MultiOpsTxnsStressTest::SecondaryKeyUpdateTxn(ThreadState* thread, Iterator* it = nullptr; long iterations = 0; - const Defer cleanup([new_c, &s, thread, &it, txn, this, &iterations]() { + const Defer cleanup([new_c, &s, thread, &txn, &it, this, &iterations]() { delete it; if (s.ok()) { thread->stats.AddIterations(iterations); @@ -719,7 +718,7 @@ Status MultiOpsTxnsStressTest::SecondaryKeyUpdateTxn(ThreadState* thread, } auto& key_gen = key_gen_for_c_[thread->tid]; key_gen->UndoAllocation(new_c); - RollbackTxn(txn).PermitUncheckedError(); + txn->Rollback().PermitUncheckedError(); }); // TODO (yanqin) try SetSnapshotOnNextOperation(). We currently need to take @@ -868,7 +867,6 @@ Status MultiOpsTxnsStressTest::SecondaryKeyUpdateTxn(ThreadState* thread, s = CommitAndCreateTimestampedSnapshotIfNeeded(thread, *txn); if (s.ok()) { - delete txn; auto& key_gen = key_gen_for_c_.at(thread->tid); key_gen->Replace(old_c, old_c_pos, new_c); } @@ -880,7 +878,7 @@ Status MultiOpsTxnsStressTest::UpdatePrimaryIndexValueTxn(ThreadState* thread, uint32_t a, uint32_t b_delta) { std::string pk_str = Record::EncodePrimaryKey(a); - Transaction* txn = nullptr; + std::unique_ptr txn; WriteOptions wopts; Status s = NewTxn(wopts, &txn); if (!s.ok()) { @@ -891,7 +889,7 @@ Status MultiOpsTxnsStressTest::UpdatePrimaryIndexValueTxn(ThreadState* thread, assert(txn); - const Defer cleanup([&s, thread, txn, this]() { + const Defer cleanup([&s, thread, &txn]() { if (s.ok()) { thread->stats.AddGets(/*ngets=*/1, /*nfounds=*/1); thread->stats.AddBytesForWrites( @@ -908,7 +906,7 @@ Status MultiOpsTxnsStressTest::UpdatePrimaryIndexValueTxn(ThreadState* thread, } else { thread->stats.AddErrors(1); } - RollbackTxn(txn).PermitUncheckedError(); + txn->Rollback().PermitUncheckedError(); }); ReadOptions ropts; ropts.rate_limiter_priority = @@ -952,9 +950,6 @@ Status MultiOpsTxnsStressTest::UpdatePrimaryIndexValueTxn(ThreadState* thread, s = CommitAndCreateTimestampedSnapshotIfNeeded(thread, *txn); - if (s.ok()) { - delete txn; - } return s; } @@ -964,7 +959,7 @@ Status MultiOpsTxnsStressTest::PointLookupTxn(ThreadState* thread, // pk may or may not exist PinnableSlice value; - Transaction* txn = nullptr; + std::unique_ptr txn; WriteOptions wopts; Status s = NewTxn(wopts, &txn); if (!s.ok()) { @@ -975,7 +970,7 @@ Status MultiOpsTxnsStressTest::PointLookupTxn(ThreadState* thread, assert(txn); - const Defer cleanup([&s, thread, txn, this]() { + const Defer cleanup([&s, thread, &txn]() { if (s.ok()) { thread->stats.AddGets(/*ngets=*/1, /*nfounds=*/1); return; @@ -984,7 +979,7 @@ Status MultiOpsTxnsStressTest::PointLookupTxn(ThreadState* thread, } else { thread->stats.AddErrors(1); } - RollbackTxn(txn).PermitUncheckedError(); + txn->Rollback().PermitUncheckedError(); }); std::shared_ptr snapshot; @@ -1001,9 +996,6 @@ Status MultiOpsTxnsStressTest::PointLookupTxn(ThreadState* thread, if (s.ok()) { s = txn->Commit(); } - if (s.ok()) { - delete txn; - } return s; } @@ -1011,7 +1003,7 @@ Status MultiOpsTxnsStressTest::RangeScanTxn(ThreadState* thread, ReadOptions ropts, uint32_t c) { std::string sk = Record::EncodeSecondaryKey(c); - Transaction* txn = nullptr; + std::unique_ptr txn; WriteOptions wopts; Status s = NewTxn(wopts, &txn); if (!s.ok()) { @@ -1022,13 +1014,13 @@ Status MultiOpsTxnsStressTest::RangeScanTxn(ThreadState* thread, assert(txn); - const Defer cleanup([&s, thread, txn, this]() { + const Defer cleanup([&s, thread, &txn]() { if (s.ok()) { thread->stats.AddIterations(1); return; } thread->stats.AddErrors(1); - RollbackTxn(txn).PermitUncheckedError(); + txn->Rollback().PermitUncheckedError(); }); std::shared_ptr snapshot; @@ -1056,10 +1048,6 @@ Status MultiOpsTxnsStressTest::RangeScanTxn(ThreadState* thread, s = iter->status(); } - if (s.ok()) { - delete txn; - } - return s; } @@ -1116,8 +1104,9 @@ void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const { Status s = record.DecodePrimaryIndexEntry(it->key(), it->value()); if (!s.ok()) { oss << "Cannot decode primary index entry " << it->key().ToString(true) - << "=>" << it->value().ToString(true); - VerificationAbort(thread->shared, oss.str(), s); + << "=>" << it->value().ToString(true) << ". Status is " + << s.ToString(); + VerificationAbort(thread->shared, oss.str()); assert(false); return; } @@ -1137,8 +1126,9 @@ void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const { std::string value; s = db_->Get(ropts, sk, &value); if (!s.ok()) { - oss << "Cannot find secondary index entry " << sk.ToString(true); - VerificationAbort(thread->shared, oss.str(), s); + oss << "Cannot find secondary index entry " << sk.ToString(true) + << ". Status is " << s.ToString(); + VerificationAbort(thread->shared, oss.str()); assert(false); return; } @@ -1165,8 +1155,9 @@ void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const { Status s = record.DecodeSecondaryIndexEntry(it->key(), it->value()); if (!s.ok()) { oss << "Cannot decode secondary index entry " - << it->key().ToString(true) << "=>" << it->value().ToString(true); - VerificationAbort(thread->shared, oss.str(), s); + << it->key().ToString(true) << "=>" << it->value().ToString(true) + << ". Status is " << s.ToString(); + VerificationAbort(thread->shared, oss.str()); assert(false); return; } @@ -1180,7 +1171,7 @@ void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const { if (!s.ok()) { oss << "Error searching pk " << Slice(pk).ToString(true) << ". " << s.ToString() << ". sk " << it->key().ToString(true); - VerificationAbort(thread->shared, oss.str(), s); + VerificationAbort(thread->shared, oss.str()); assert(false); return; } @@ -1188,8 +1179,8 @@ void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const { s = std::get<0>(result); if (!s.ok()) { oss << "Error decoding primary index value " - << Slice(value).ToString(true) << ". " << s.ToString(); - VerificationAbort(thread->shared, oss.str(), s); + << Slice(value).ToString(true) << ". Status is " << s.ToString(); + VerificationAbort(thread->shared, oss.str()); assert(false); return; } @@ -1199,7 +1190,7 @@ void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const { << Slice(value).ToString(true) << " (a=" << record.a_value() << ", c=" << c_in_primary << "), sk: " << it->key().ToString(true) << " (c=" << record.c_value() << ")"; - VerificationAbort(thread->shared, oss.str(), s); + VerificationAbort(thread->shared, oss.str()); assert(false); return; } @@ -1210,7 +1201,7 @@ void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const { oss << "Pk/sk mismatch: primary index has " << primary_index_entries_count << " entries. Secondary index has " << secondary_index_entries_count << " entries."; - VerificationAbort(thread->shared, oss.str(), Status::OK()); + VerificationAbort(thread->shared, oss.str()); assert(false); return; } diff --git a/db_stress_tool/no_batched_ops_stress.cc b/db_stress_tool/no_batched_ops_stress.cc index de03e9795..0e6efbbc3 100644 --- a/db_stress_tool/no_batched_ops_stress.cc +++ b/db_stress_tool/no_batched_ops_stress.cc @@ -9,6 +9,7 @@ #include "db_stress_tool/expected_state.h" #ifdef GFLAGS +#include "db/wide/wide_columns_helper.h" #include "db_stress_tool/db_stress_common.h" #include "rocksdb/utilities/transaction_db.h" #include "utilities/fault_injection_fs.h" @@ -16,9 +17,9 @@ namespace ROCKSDB_NAMESPACE { class NonBatchedOpsStressTest : public StressTest { public: - NonBatchedOpsStressTest() {} + NonBatchedOpsStressTest() = default; - virtual ~NonBatchedOpsStressTest() {} + virtual ~NonBatchedOpsStressTest() = default; void VerifyDb(ThreadState* thread) const override { // This `ReadOptions` is for validation purposes. Ignore @@ -165,9 +166,8 @@ class NonBatchedOpsStressTest : public StressTest { if (s.ok()) { const WideColumns& columns = result.columns(); - if (!columns.empty() && - columns.front().name() == kDefaultWideColumnName) { - from_db = columns.front().value().ToString(); + if (WideColumnsHelper::HasDefaultColumn(columns)) { + from_db = WideColumnsHelper::GetDefaultColumn(columns).ToString(); } if (!VerifyWideColumns(columns)) { @@ -251,9 +251,9 @@ class NonBatchedOpsStressTest : public StressTest { if (statuses[j].ok()) { const WideColumns& columns = results[j].columns(); - if (!columns.empty() && - columns.front().name() == kDefaultWideColumnName) { - from_db = columns.front().value().ToString(); + if (WideColumnsHelper::HasDefaultColumn(columns)) { + from_db = + WideColumnsHelper::GetDefaultColumn(columns).ToString(); } if (!VerifyWideColumns(columns)) { @@ -442,7 +442,7 @@ class NonBatchedOpsStressTest : public StressTest { if (!s.ok()) { fprintf(stderr, "dropping column family error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } s = db_->CreateColumnFamily(ColumnFamilyOptions(options_), new_name, &column_families_[cf]); @@ -451,7 +451,7 @@ class NonBatchedOpsStressTest : public StressTest { if (!s.ok()) { fprintf(stderr, "creating column family error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } thread->shared->UnlockColumnFamily(cf); } @@ -585,6 +585,7 @@ class NonBatchedOpsStressTest : public StressTest { bool do_consistency_check = thread->rand.OneIn(4); ReadOptions readoptionscopy = read_opts; + if (do_consistency_check) { readoptionscopy.snapshot = db_->GetSnapshot(); } @@ -603,7 +604,7 @@ class NonBatchedOpsStressTest : public StressTest { // Create a transaction in order to write some data. The purpose is to // exercise WriteBatchWithIndex::MultiGetFromBatchAndDB. The transaction // will be rolled back once MultiGet returns. - Transaction* txn = nullptr; + std::unique_ptr txn; if (use_txn) { WriteOptions wo; if (FLAGS_rate_limit_auto_wal_flush) { @@ -611,8 +612,8 @@ class NonBatchedOpsStressTest : public StressTest { } Status s = NewTxn(wo, &txn); if (!s.ok()) { - fprintf(stderr, "NewTxn: %s\n", s.ToString().c_str()); - std::terminate(); + fprintf(stderr, "NewTxn error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); } } for (size_t i = 0; i < num_keys; ++i) { @@ -623,7 +624,7 @@ class NonBatchedOpsStressTest : public StressTest { if (!shared->AllowsOverwrite(rand_key) && shared->Exists(column_family, rand_key)) { // Just do read your write checks for keys that allow overwrites. - ryw_expected_values.push_back(std::nullopt); + ryw_expected_values.emplace_back(std::nullopt); continue; } // With a 1 in 10 probability, insert the just added key in the batch @@ -661,11 +662,12 @@ class NonBatchedOpsStressTest : public StressTest { assert(false); } if (!s.ok()) { - fprintf(stderr, "Transaction put: %s\n", s.ToString().c_str()); - std::terminate(); + fprintf(stderr, "Transaction put error: %s\n", + s.ToString().c_str()); + thread->shared->SafeTerminate(); } } else { - ryw_expected_values.push_back(std::nullopt); + ryw_expected_values.emplace_back(std::nullopt); } } } @@ -778,9 +780,17 @@ class NonBatchedOpsStressTest : public StressTest { if (use_txn) { assert(txn); + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_GET); tmp_s = txn->Get(readoptionscopy, cfh, key, &value); + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_MULTIGET); } else { + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_GET); tmp_s = db_->Get(readoptionscopy, cfh, key, &value); + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_MULTIGET); } if (!tmp_s.ok() && !tmp_s.IsNotFound()) { fprintf(stderr, "Get error: %s\n", s.ToString().c_str()); @@ -866,7 +876,7 @@ class NonBatchedOpsStressTest : public StressTest { db_->ReleaseSnapshot(readoptionscopy.snapshot); } if (use_txn) { - RollbackTxn(txn); + txn->Rollback().PermitUncheckedError(); } return statuses; } @@ -1058,7 +1068,8 @@ class NonBatchedOpsStressTest : public StressTest { is_consistent = false; } else if (check_get_entity && (s.ok() || s.IsNotFound())) { PinnableWideColumns cmp_result; - + ThreadStatusUtil::SetThreadOperation( + ThreadStatus::OperationType::OP_GETENTITY); const Status cmp_s = db_->GetEntity(read_opts_copy, cfh, key_slices[i], &cmp_result); @@ -1267,10 +1278,13 @@ class NonBatchedOpsStressTest : public StressTest { const size_t sz = GenerateValue(value_base, value, sizeof(value)); const Slice v(value, sz); - Status s; - if (FLAGS_use_merge) { + if (FLAGS_use_put_entity_one_in > 0 && + (value_base % FLAGS_use_put_entity_one_in) == 0) { + s = db_->PutEntity(write_opts, cfh, k, + GenerateWideColumns(value_base, v)); + } else if (FLAGS_use_merge) { if (!FLAGS_use_txn) { if (FLAGS_user_timestamp_size == 0) { s = db_->Merge(write_opts, cfh, k, v); @@ -1278,19 +1292,10 @@ class NonBatchedOpsStressTest : public StressTest { s = db_->Merge(write_opts, cfh, k, write_ts, v); } } else { - Transaction* txn; - s = NewTxn(write_opts, &txn); - if (s.ok()) { - s = txn->Merge(cfh, k, v); - if (s.ok()) { - s = CommitTxn(txn, thread); - } - } + s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) { + return txn.Merge(cfh, k, v); + }); } - } else if (FLAGS_use_put_entity_one_in > 0 && - (value_base % FLAGS_use_put_entity_one_in) == 0) { - s = db_->PutEntity(write_opts, cfh, k, - GenerateWideColumns(value_base, v)); } else { if (!FLAGS_use_txn) { if (FLAGS_user_timestamp_size == 0) { @@ -1299,34 +1304,27 @@ class NonBatchedOpsStressTest : public StressTest { s = db_->Put(write_opts, cfh, k, write_ts, v); } } else { - Transaction* txn; - s = NewTxn(write_opts, &txn); - if (s.ok()) { - s = txn->Put(cfh, k, v); - if (s.ok()) { - s = CommitTxn(txn, thread); - } - } + s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) { + return txn.Put(cfh, k, v); + }); } } - pending_expected_value.Commit(); - if (!s.ok()) { - if (FLAGS_injest_error_severity >= 2) { + if (FLAGS_inject_error_severity >= 2) { if (!is_db_stopped_ && s.severity() >= Status::Severity::kFatalError) { is_db_stopped_ = true; } else if (!is_db_stopped_ || s.severity() < Status::Severity::kFatalError) { fprintf(stderr, "put or merge error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } } else { fprintf(stderr, "put or merge error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } } - + pending_expected_value.Commit(); thread->stats.AddBytesForWrites(1, sz); PrintKeyValue(rand_column_family, static_cast(rand_key), value, sz); @@ -1364,33 +1362,28 @@ class NonBatchedOpsStressTest : public StressTest { s = db_->Delete(write_opts, cfh, key, write_ts); } } else { - Transaction* txn; - s = NewTxn(write_opts, &txn); - if (s.ok()) { - s = txn->Delete(cfh, key); - if (s.ok()) { - s = CommitTxn(txn, thread); - } - } + s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) { + return txn.Delete(cfh, key); + }); } - pending_expected_value.Commit(); - thread->stats.AddDeletes(1); if (!s.ok()) { - if (FLAGS_injest_error_severity >= 2) { + if (FLAGS_inject_error_severity >= 2) { if (!is_db_stopped_ && s.severity() >= Status::Severity::kFatalError) { is_db_stopped_ = true; } else if (!is_db_stopped_ || s.severity() < Status::Severity::kFatalError) { fprintf(stderr, "delete error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } } else { fprintf(stderr, "delete error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } } + pending_expected_value.Commit(); + thread->stats.AddDeletes(1); } else { PendingExpectedValue pending_expected_value = shared->PrepareSingleDelete(rand_column_family, rand_key); @@ -1401,32 +1394,28 @@ class NonBatchedOpsStressTest : public StressTest { s = db_->SingleDelete(write_opts, cfh, key, write_ts); } } else { - Transaction* txn; - s = NewTxn(write_opts, &txn); - if (s.ok()) { - s = txn->SingleDelete(cfh, key); - if (s.ok()) { - s = CommitTxn(txn, thread); - } - } + s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) { + return txn.SingleDelete(cfh, key); + }); } - pending_expected_value.Commit(); - thread->stats.AddSingleDeletes(1); + if (!s.ok()) { - if (FLAGS_injest_error_severity >= 2) { + if (FLAGS_inject_error_severity >= 2) { if (!is_db_stopped_ && s.severity() >= Status::Severity::kFatalError) { is_db_stopped_ = true; } else if (!is_db_stopped_ || s.severity() < Status::Severity::kFatalError) { fprintf(stderr, "single delete error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } } else { fprintf(stderr, "single delete error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } } + pending_expected_value.Commit(); + thread->stats.AddSingleDeletes(1); } return s; } @@ -1475,17 +1464,17 @@ class NonBatchedOpsStressTest : public StressTest { s = db_->DeleteRange(write_opts, cfh, key, end_key); } if (!s.ok()) { - if (FLAGS_injest_error_severity >= 2) { + if (FLAGS_inject_error_severity >= 2) { if (!is_db_stopped_ && s.severity() >= Status::Severity::kFatalError) { is_db_stopped_ = true; } else if (!is_db_stopped_ || s.severity() < Status::Severity::kFatalError) { fprintf(stderr, "delete range error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } } else { fprintf(stderr, "delete range error: %s\n", s.ToString().c_str()); - std::terminate(); + thread->shared->SafeTerminate(); } } for (PendingExpectedValue& pending_expected_value : @@ -1549,9 +1538,18 @@ class NonBatchedOpsStressTest : public StressTest { pending_expected_values.push_back(pending_expected_value); char value[100]; - size_t value_len = GenerateValue(value_base, value, sizeof(value)); auto key_str = Key(key); - s = sst_file_writer.Put(Slice(key_str), Slice(value, value_len)); + const size_t value_len = GenerateValue(value_base, value, sizeof(value)); + const Slice k(key_str); + const Slice v(value, value_len); + + if (FLAGS_use_put_entity_one_in > 0 && + (value_base % FLAGS_use_put_entity_one_in) == 0) { + WideColumns columns = GenerateWideColumns(value_base, v); + s = sst_file_writer.PutEntity(k, columns); + } else { + s = sst_file_writer.Put(k, v); + } } if (s.ok() && keys.empty()) { @@ -1566,12 +1564,16 @@ class NonBatchedOpsStressTest : public StressTest { {sst_filename}, IngestExternalFileOptions()); } if (!s.ok()) { - fprintf(stderr, "file ingestion error: %s\n", s.ToString().c_str()); - std::terminate(); - } - - for (size_t i = 0; i < pending_expected_values.size(); ++i) { - pending_expected_values[i].Commit(); + if (!s.IsIOError() || !std::strstr(s.getState(), "injected")) { + fprintf(stderr, "file ingestion error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } else { + fprintf(stdout, "file ingestion error: %s\n", s.ToString().c_str()); + } + } else { + for (size_t i = 0; i < pending_expected_values.size(); ++i) { + pending_expected_values[i].Commit(); + } } } @@ -1600,8 +1602,6 @@ class NonBatchedOpsStressTest : public StressTest { const int64_t ub = lb + num_iter; - // Lock the whole range over which we might iterate to ensure it doesn't - // change under us. const int rand_column_family = rand_column_families[0]; // Testing parallel read and write to the same key with user timestamp @@ -1612,7 +1612,9 @@ class NonBatchedOpsStressTest : public StressTest { } ReadOptions ro(read_opts); - ro.total_order_seek = true; + if (FLAGS_prefix_size > 0) { + ro.total_order_seek = true; + } std::string read_ts_str; Slice read_ts; @@ -1680,6 +1682,7 @@ class NonBatchedOpsStressTest : public StressTest { }; auto check_no_key_in_range = [&](int64_t start, int64_t end) { + assert(start <= end); for (auto j = std::max(start, lb); j < std::min(end, ub); ++j) { std::size_t index = static_cast(j - lb); assert(index < pre_read_expected_values.size() && @@ -1694,11 +1697,14 @@ class NonBatchedOpsStressTest : public StressTest { thread->shared->SetVerificationFailure(); if (iter->Valid()) { fprintf(stderr, - "Expected state has key %s, iterator is at key %s\n", + "Verification failed. Expected state has key %s, iterator " + "is at key %s\n", Slice(Key(j)).ToString(true).c_str(), iter->key().ToString(true).c_str()); } else { - fprintf(stderr, "Expected state has key %s, iterator is invalid\n", + fprintf(stderr, + "Verification failed. Expected state has key %s, iterator " + "is invalid\n", Slice(Key(j)).ToString(true).c_str()); } fprintf(stderr, "Column family: %s, op_logs: %s\n", @@ -1722,6 +1728,7 @@ class NonBatchedOpsStressTest : public StressTest { uint64_t curr = 0; while (true) { + assert(last_key < ub); if (!iter->Valid()) { if (!iter->status().ok()) { thread->shared->SetVerificationFailure(); @@ -1744,6 +1751,19 @@ class NonBatchedOpsStressTest : public StressTest { // iter is valid, the range (last_key, current key) was skipped GetIntVal(iter->key().ToString(), &curr); + if (static_cast(curr) <= last_key) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "TestIterateAgainstExpected failed: found unexpectedly small " + "key\n"); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + fprintf(stderr, "Last op found key: %s, expected at least: %s\n", + Slice(Key(curr)).ToString(true).c_str(), + Slice(Key(last_key + 1)).ToString(true).c_str()); + thread->stats.AddErrors(1); + return Status::OK(); + } if (!check_no_key_in_range(last_key + 1, static_cast(curr))) { return Status::OK(); } @@ -1766,6 +1786,7 @@ class NonBatchedOpsStressTest : public StressTest { last_key = ub; while (true) { + assert(lb < last_key); if (!iter->Valid()) { if (!iter->status().ok()) { thread->shared->SetVerificationFailure(); @@ -1788,6 +1809,19 @@ class NonBatchedOpsStressTest : public StressTest { // the range (current key, last key) was skipped GetIntVal(iter->key().ToString(), &curr); + if (last_key <= static_cast(curr)) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "TestIterateAgainstExpected failed: found unexpectedly large " + "key\n"); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + fprintf(stderr, "Last op found key: %s, expected at most: %s\n", + Slice(Key(curr)).ToString(true).c_str(), + Slice(Key(last_key - 1)).ToString(true).c_str()); + thread->stats.AddErrors(1); + return Status::OK(); + } if (!check_no_key_in_range(static_cast(curr + 1), last_key)) { return Status::OK(); } @@ -1802,7 +1836,9 @@ class NonBatchedOpsStressTest : public StressTest { op_logs += "P"; } - if (thread->rand.OneIn(2)) { + // Write-prepared and Write-unprepared do not support Refresh() yet. + if (!(FLAGS_use_txn && FLAGS_txn_write_policy != 0) && + thread->rand.OneIn(2)) { pre_read_expected_values.clear(); post_read_expected_values.clear(); // Refresh after forward/backward scan to allow higher chance of SV @@ -1811,7 +1847,9 @@ class NonBatchedOpsStressTest : public StressTest { pre_read_expected_values.push_back( shared->Get(rand_column_family, i + lb)); } - iter->Refresh(); + Status rs = iter->Refresh(); + assert(rs.ok()); + op_logs += "Refresh "; for (int64_t i = 0; i < static_cast(expected_values_size); ++i) { post_read_expected_values.push_back( shared->Get(rand_column_family, i + lb)); @@ -1836,6 +1874,21 @@ class NonBatchedOpsStressTest : public StressTest { if (!check_no_key_in_range(mid, ub)) { return Status::OK(); } + } else if (iter->Valid()) { + GetIntVal(iter->key().ToString(), &curr); + if (static_cast(curr) < mid) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "TestIterateAgainstExpected failed: found unexpectedly small " + "key\n"); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + fprintf(stderr, "Last op found key: %s, expected at least: %s\n", + Slice(Key(curr)).ToString(true).c_str(), + Slice(Key(mid)).ToString(true).c_str()); + thread->stats.AddErrors(1); + return Status::OK(); + } } } else { iter->SeekForPrev(key); @@ -1845,6 +1898,21 @@ class NonBatchedOpsStressTest : public StressTest { if (!check_no_key_in_range(lb, mid + 1)) { return Status::OK(); } + } else if (iter->Valid()) { + GetIntVal(iter->key().ToString(), &curr); + if (mid < static_cast(curr)) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "TestIterateAgainstExpected failed: found unexpectedly large " + "key\n"); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + fprintf(stderr, "Last op found key: %s, expected at most: %s\n", + Slice(Key(curr)).ToString(true).c_str(), + Slice(Key(mid)).ToString(true).c_str()); + thread->stats.AddErrors(1); + return Status::OK(); + } } } @@ -1876,7 +1944,9 @@ class NonBatchedOpsStressTest : public StressTest { post_read_expected_value)) { // Fail fast to preserve the DB state. thread->shared->SetVerificationFailure(); - fprintf(stderr, "Iterator has key %s, but expected state does not.\n", + fprintf(stderr, + "Verification failed: iterator has key %s, but expected " + "state does not.\n", iter->key().ToString(true).c_str()); fprintf(stderr, "Column family: %s, op_logs: %s\n", cfh->GetName().c_str(), op_logs.c_str()); @@ -1892,6 +1962,19 @@ class NonBatchedOpsStressTest : public StressTest { } uint64_t next = 0; GetIntVal(iter->key().ToString(), &next); + if (next <= curr) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "TestIterateAgainstExpected failed: found unexpectedly " + "small key\n"); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + fprintf(stderr, "Last op found key: %s, expected at least: %s\n", + Slice(Key(next)).ToString(true).c_str(), + Slice(Key(curr + 1)).ToString(true).c_str()); + thread->stats.AddErrors(1); + return Status::OK(); + } if (!check_no_key_in_range(static_cast(curr + 1), static_cast(next))) { return Status::OK(); @@ -1904,6 +1987,19 @@ class NonBatchedOpsStressTest : public StressTest { } uint64_t prev = 0; GetIntVal(iter->key().ToString(), &prev); + if (curr <= prev) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "TestIterateAgainstExpected failed: found unexpectedly " + "large key\n"); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + fprintf(stderr, "Last op found key: %s, expected at most: %s\n", + Slice(Key(prev)).ToString(true).c_str(), + Slice(Key(curr - 1)).ToString(true).c_str()); + thread->stats.AddErrors(1); + return Status::OK(); + } if (!check_no_key_in_range(static_cast(prev + 1), static_cast(curr))) { return Status::OK(); @@ -1953,9 +2049,7 @@ class NonBatchedOpsStressTest : public StressTest { const Slice slice(value_from_db); const uint32_t value_base_from_db = GetValueBase(slice); if (ExpectedValueHelper::MustHaveNotExisted(expected_value, - expected_value) || - !ExpectedValueHelper::InExpectedValueBaseRange( - value_base_from_db, expected_value, expected_value)) { + expected_value)) { VerificationAbort(shared, msg_prefix + ": Unexpected value found", cf, key, value_from_db, ""); return false; @@ -1964,6 +2058,14 @@ class NonBatchedOpsStressTest : public StressTest { size_t expected_value_data_size = GenerateValue(expected_value.GetValueBase(), expected_value_data, sizeof(expected_value_data)); + if (!ExpectedValueHelper::InExpectedValueBaseRange( + value_base_from_db, expected_value, expected_value)) { + VerificationAbort(shared, msg_prefix + ": Unexpected value found", cf, + key, value_from_db, + Slice(expected_value_data, expected_value_data_size)); + return false; + } + // TODO: are the length/memcmp() checks repetitive? if (value_from_db.length() != expected_value_data_size) { VerificationAbort(shared, msg_prefix + ": Length of value read is not equal", diff --git a/docs/_posts/2023-11-06-java-jni-benchmarks.markdown b/docs/_posts/2023-11-06-java-jni-benchmarks.markdown new file mode 100644 index 000000000..2cf5c8362 --- /dev/null +++ b/docs/_posts/2023-11-06-java-jni-benchmarks.markdown @@ -0,0 +1,287 @@ +--- +title: Java API Performance Improvements +layout: post +author: alanpaxton +category: blog +--- +# RocksDB Java API Performance Improvements + +Evolved Binary has been working on several aspects of how the Java API to RocksDB can be improved. Two aspects of this which are of particular importance are performance and the developer experience. + +* We have built some synthetic benchmark code to determine which are the most efficient methods of transferring data between Java and C++. +* We have used the results of the synthetic benchmarking to guide plans for rationalising the API interfaces. +* We have made some opportunistic performance optimizations/fixes within the Java API which have already yielded noticable improvements. + +## Synthetic JNI API Performance Benchmarks +The synthetic benchmark repository contains tests designed to isolate the Java to/from C++ interaction of a canonical data intensive Key/Value Store implemented in C++ with a Java (JNI) API layered on top. + +JNI provides several mechanisms for allowing transfer of data between Java buffers and C++ buffers. These mechanisms are not trivial, because they require the JNI system to ensure that Java memory under the control of the JVM is not moved or garbage collected whilst it is being accessed outside the direct control of the JVM. + +We set out to determine which of multiple options for transfer of data from `C++` to `Java` and vice-versa were the most efficient. We used the [Java Microbenchmark Harness](https://github.com/openjdk/jmh) to set up repeatable benchmarks to measure all the options. + +We explore these and some other potential mechanisms in the detailed results (in our [Synthetic JNI performance repository](https://github.com/evolvedbinary/jni-benchmarks/blob/main/DataBenchmarks.md)) + +We summarise this work here: + +### The Model + +* In `C++` we represent the on-disk data as an in-memory map of `(key, value)` + pairs. +* For a fetch query, we expect the result to be a Java object with access to the + contents of the _value_. This may be a standard Java object which does the job + of data access (a `byte[]` or a `ByteBuffer`) or an object of our own devising + which holds references to the value in some form (a `FastBuffer` pointing to + `com.sun.unsafe.Unsafe` unsafe memory, for instance). + +### Data Types + +There are several potential data types for holding data for transfer, and they +are unsurprisingly quite connected underneath. + +#### Byte Array + +The simplest data container is a _raw_ array of bytes (`byte[]`). + +There are 3 different mechanisms for transferring data between a `byte[]` and +C++ + +* At the C++ side, the method + [`JNIEnv.GetArrayCritical()`](https://docs.oracle.com/en/java/javase/13/docs/specs/jni/functions.html#getprimitivearraycritical) + allows access to a C++ pointer to the underlying array. +* The `JNIEnv` methods `GetByteArrayElements()` and `ReleaseByteArrayElements()` + fetch references/copies to and from the contents of a byte array, with less + concern for critical sections than the _critical_ methods, though they are + consequently more likely/certain to result in (extra) copies. +* The `JNIEnv` methods `GetByteArrayRegion()` and `SetByteArrayRegion()` + transfer raw C++ buffer data to and from the contents of a byte array. These + must ultimately do some data pinning for the duration of copies; the + mechanisms may be similar or different to the _critical_ operations, and + therefore performance may differ. + +#### Byte Buffer + +A `ByteBuffer` abstracts the contents of a collection of bytes, and was in fact +introduced to support a range of higher-performance I/O operations in some +circumstances. + +There are 2 types of byte buffers in Java, _indirect_ and _direct_. Indirect +byte buffers are the standard, and the memory they use is on-heap as with all +usual Java objects. In contrast, direct byte buffers are used to wrap off-heap +memory which is accessible to direct network I/O. Either type of `ByteBuffer` +can be allocated at the Java side, using the `allocate()` and `allocateDirect()` +methods respectively. + +Direct byte buffers can be created in C++ using the JNI method +[`JNIEnv.NewDirectByteBuffer()`](https://docs.oracle.com/en/java/javase/13/docs/specs/jni/functions.html#newdirectbytebuffer) +to wrap some native (C++) memory. + +Direct byte buffers can be accessed in C++ using the +[`JNIEnv.GetDirectBufferAddress()`](https://docs.oracle.com/en/java/javase/13/docs/specs/jni/functions.html#GetDirectBufferAddress) +and measured using +[`JNIEnv.GetDirectBufferCapacity()`](https://docs.oracle.com/en/java/javase/13/docs/specs/jni/functions.html#GetDirectBufferCapacity) + +#### Unsafe Memory + +The call `com.sun.unsafe.Unsafe.allocateMemory()` returns a handle which is (of course) just a pointer to raw memory, and +can be used as such on the C++ side. We could turn it into a byte buffer on the +C++ side by calling `JNIEnv.NewDirectByteBuffer()`, or simply use it as a native +C++ buffer at the expected address, assuming we record or remember how much +space was allocated. + +A custom `FastBuffer` class provides access to unsafe memory from the Java side. + + +#### Allocation + +For these benchmarks, allocation has been excluded from the benchmark costs by +pre-allocating a quantity of buffers of the appropriate kind as part of the test +setup. Each run of the benchmark acquires an existing buffer from a pre-allocated +FIFO list, and returns it afterwards. A small test has +confirmed that the request and return cycle is of insignificant cost compared to +the benchmark API call. + +### GetJNIBenchmark Performance + +Benchmarks ran for a duration of order 6 hours on an otherwise unloaded VM, + the error bars are small and we can have strong confidence in the values + derived and plotted. + +![Raw JNI Get small](/static/images/jni-get-benchmarks/fig_1024_1_none_nopoolbig.png) + +Comparing all the benchmarks as the data size tends large, the conclusions we +can draw are: + +- Indirect byte buffers add cost; they are effectively an overhead on plain + `byte[]` and the JNI-side only allows them to be accessed via their + encapsulated `byte[]`. +- `SetRegion` and `GetCritical` mechanisms for copying data into a `byte[]` are + of very comparable performance; presumably the behaviour behind the scenes of + `SetRegion` is very similar to that of declaring a critical region, doing a + `memcpy()` and releasing the critical region. +- `GetElements` methods for transferring data from C++ to Java are consistently + less efficient than `SetRegion` and `GetCritical`. +- Getting into a raw memory buffer, passed as an address (the `handle` of an + `Unsafe` or of a netty `ByteBuf`) is of similar cost to the more efficient + `byte[]` operations. +- Getting into a direct `nio.ByteBuffer` is of similar cost again; while the + ByteBuffer is passed over JNI as an ordinary Java object, JNI has a specific + method for getting hold of the address of the direct buffer, and using this, the + `get()` cost with a ByteBuffer is just that of the underlying C++ `memcpy()`. + +At small(er) data sizes, we can see whether other factors are important. + +![Raw JNI Get large](/static/images/jni-get-benchmarks/fig_1024_1_none_nopoolsmall.png) + +- Indirect byte buffers are the most significant overhead here. Again, we can + conclude that this is due to pure overhead compared to `byte[]` operations. +- At the lowest data sizes, netty `ByteBuf`s and unsafe memory are marginally + more efficient than `byte[]`s or (slightly less efficient) direct + `nio.Bytebuffer`s. This may be explained by even the small cost of + calling the JNI model on the C++ side simply to acquire a + direct buffer address. The margins (nanoseconds) here are extremely small. + +#### Post processing the results + +Our benchmark model for post-processing is to transfer the results into a +`byte[]`. Where the result is already a `byte[]` this may seem like an unfair +extra cost, but the aim is to model the least cost processing step for any kind +of result. + +- Copying into a `byte[]` using the bulk methods supported by `byte[]`, + `nio.ByteBuffer` have comparable performance. +- Accessing the contents of an `Unsafe` buffer using the supplied unsafe methods + is inefficient. The access is word by + word, in Java. +- Accessing the contents of a netty `ByteBuf` is similarly inefficient; again + the access is presumably word by word, using normal + Java mechanisms. + +![Copy out JNI Get](/static/images/jni-get-benchmarks/fig_1024_1_copyout_nopoolbig.png) + +### PutJNIBenchmark + +We benchmarked `Put` methods in a similar synthetic fashion in less depth, but enough to confirm that the performance profile is similar/symmetrical. As with `get()` using `GetElements` is the least performant way of implementing transfers to/from Java objects in C++/JNI, and other JNI mechanisms do not differ greatly one from another. + +## Lessons from Synthetic API + +Performance analysis shows that for `get()`, fetching into allocated `byte[]` is +equally as efficient as any other mechanism, as long as JNI region methods are used +for the internal data transfer. Copying out or otherwise using the +result on the Java side is straightforward and efficient. Using `byte[]` avoids the manual memory +management required with direct `nio.ByteBuffer`s, which extra work does not +appear to provide any gain. A C++ implementation using the `GetRegion` JNI +method is probably to be preferred to using `GetCritical` because while their +performance is equal, `GetRegion` is a higher-level/simpler abstraction. + +Vitally, whatever JNI transfer mechanism is chosen, the buffer allocation +mechanism and pattern is crucial to achieving good performance. We experimented +with making use of netty's pooled allocator part of the benchmark, and the +difference of `getIntoPooledNettyByteBuf`, using the allocator, compared to +`getIntoNettyByteBuf` using the same pre-allocate on setup as every other +benchmark, is significant. + +Equally importantly, transfer of data to or from buffers should where possible +be done in bulk, using array copy or buffer copy mechanisms. Thought should +perhaps be given to supporting common transformations in the underlying C++ +layer. + +## API Recommendations + +Of course there is some noise within the results. but we can agree: + + * Don't make copies you don't need to make + * Don't allocate/deallocate when you can avoid it + +Translating this into designing an efficient API, we want to: + + * Support API methods that return results in buffers supplied by the client. + * Support `byte[]`-based APIs as the simplest way of getting data into a usable configuration for a broad range of Java use. + * Support direct `ByteBuffer`s as these can reduce copies when used as part of a chain of `ByteBuffer`-based operations. This sort of sophisticated streaming model is most likely to be used by clients where performance is important, and so we decide to support it. + * Support indirect `ByteBuffer`s for a combination of reasons: + * API consistency between direct and indirect buffers + * Simplicity of implementation, as we can wrap `byte[]`-oriented methods + * Continue to support methods which allocate return buffers per-call, as these are the easiest to use on initial encounter with the RocksDB API. + +High performance Java interaction with RocksDB ultimately requires architectural decisions by the client + * Use more complex (client supplied buffer) API methods where performance matters + * Don't allocate/deallocate where you don't need to + * recycle your own buffers where this makes sense + * or make sure that you are supplying the ultimate destination buffer (your cache, or a target network buffer) as input to RocksDB `get()` and `put()` calls + +We are currently implementing a number of extra methods consistently across the Java fetch and store APIs to RocksDB in the PR [Java API consistency between RocksDB.put() , .merge() and Transaction.put() , .merge()](https://github.com/facebook/rocksdb/pull/11019) according to these principles. + +## Optimizations + +### Reduce Copies within API Implementation + +Having analysed JNI performance as described, we reviewed the core of RocksJNI for opportunities to improve the performance. We noticed one thing in particular; some of the `get()` methods of the Java API had not been updated to take advantage of the new [`PinnableSlice`](http://rocksdb.org/blog/2017/08/24/pinnableslice.html) methods. + +Fixing this turned out to be a straightforward change, which has now been incorporated in the codebase [Improve Java API `get()` performance by reducing copies](https://github.com/facebook/rocksdb/pull/10970) + +#### Performance Results + +Using the JMH performances tests we updated as part of the above PR, we can see a small but consistent improvement in performance for all of the different get method variants which we have enhanced in the PR. + +```sh +java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000,50000 -p keySize=128 -p valueSize=1024,16384 -p columnFamilyTestType="1_column_family" GetBenchmarks.get GetBenchmarks.preallocatedByteBufferGet GetBenchmarks.preallocatedGet +``` +The y-axis shows `ops/sec` in throughput, so higher is better. + +![](/static/images/jni-get-benchmarks/optimization-graph.png) + +### Analysis + +Before the invention of the Pinnable Slice the simplest RocksDB (native) API `Get()` looked like this: + +```cpp +Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value) +``` + +After PinnableSlice the correct way for new code to implement a `get()` is like this + +```cpp +Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) +``` + +But of course RocksDB has to support legacy code, so there is an `inline` method in `db.h` which re-implements the former using the latter. +And RocksJava API implementation seamlessly continues to use the `std::string`-based `get()` + +Let's examine what happens when get() is called from Java + +```cpp +jint Java_org_rocksdb_RocksDB_get__JJ_3BII_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jlong jropt_handle, jbyteArray jkey, + jint jkey_off, jint jkey_len, jbyteArray jval, jint jval_off, jint jval_len, + jlong jcf_handle) +``` + + 1. Create an empty `std::string value` + 2. Call `DB::Get()` using the `std::string` variant + 3. Copy the resultant `std::string` into Java, using the JNI `SetByteArrayRegion()` method + +So stage (3) costs us a copy into Java. It's mostly unavoidable that there will be at least the one copy from a C++ buffer into a Java buffer. + +But what does stage 2 do ? + + * Create a `PinnableSlice(std::string&)` which uses the value as the slice's backing buffer. + * Call `DB::Get()` using the PinnableSlice variant + * Work out if the slice has pinned data, in which case copy the pinned data into value and release it. + * ..or, if the slice has not pinned data, it is already in value (because we tried, but couldn't pin anything). + +So stage (2) costs us a copy into a `std::string`. But! It's just a naive `std::string` that we have copied a large buffer into. And in RocksDB, the buffer is or can be large, so an extra copy something we need to worry about. + +Luckily this is easy to fix. In the Java API (JNI) implementation: + + 1. Create a PinnableSlice() which uses its own default backing buffer. + 2. Call `DB::Get()` using the `PinnableSlice` variant of the RocksDB API + 3. Copy the data indicated by the `PinnableSlice` straight into the Java output buffer using the JNI `SetByteArrayRegion()` method, then release the slice. + 4. Work out if the slice has successfully pinned data, in which case copy the pinned data straight into the Java output buffer using the JNI `SetByteArrayRegion()` method, then release the pin. + 5. ..or, if the slice has not pinned data, it is in the pinnable slice's default backing buffer. All that is left, is to copy it straight into the Java output buffer using the JNI SetByteArrayRegion() method. + +In the case where the `PinnableSlice` has succesfully pinned the data, this saves us the intermediate copy to the `std::string`. In the case where it hasn't, we still have the extra copy so the observed performance improvement depends on when the data can be pinned. Luckily, our benchmarking suggests that the pin is happening in a significant number of cases. + +On discussion with the RocksDB core team we understand that the core `PinnableSlice` optimization is most likely to succeed when pages are loaded from the block cache, rather than when they are in `memtable`. And it might be possible to successfully pin in the `memtable` as well, with some extra coding effort. This would likely improve the results for these benchmarks. diff --git a/docs/static/images/jni-get-benchmarks/fig_1024_1_copyout_nopoolbig.png b/docs/static/images/jni-get-benchmarks/fig_1024_1_copyout_nopoolbig.png new file mode 100644 index 000000000..2d662dde8 Binary files /dev/null and b/docs/static/images/jni-get-benchmarks/fig_1024_1_copyout_nopoolbig.png differ diff --git a/docs/static/images/jni-get-benchmarks/fig_1024_1_none_nopoolbig.png b/docs/static/images/jni-get-benchmarks/fig_1024_1_none_nopoolbig.png new file mode 100644 index 000000000..ef071b388 Binary files /dev/null and b/docs/static/images/jni-get-benchmarks/fig_1024_1_none_nopoolbig.png differ diff --git a/docs/static/images/jni-get-benchmarks/fig_1024_1_none_nopoolsmall.png b/docs/static/images/jni-get-benchmarks/fig_1024_1_none_nopoolsmall.png new file mode 100644 index 000000000..732c42ff7 Binary files /dev/null and b/docs/static/images/jni-get-benchmarks/fig_1024_1_none_nopoolsmall.png differ diff --git a/docs/static/images/jni-get-benchmarks/optimization-graph.png b/docs/static/images/jni-get-benchmarks/optimization-graph.png new file mode 100644 index 000000000..8eb1e6bcf Binary files /dev/null and b/docs/static/images/jni-get-benchmarks/optimization-graph.png differ diff --git a/env/composite_env.cc b/env/composite_env.cc index 8ddc9a1a6..59434785c 100644 --- a/env/composite_env.cc +++ b/env/composite_env.cc @@ -504,7 +504,7 @@ EnvWrapper::EnvWrapper(const std::shared_ptr& t) : target_(t) { RegisterOptions("", &target_, &env_wrapper_type_info); } -EnvWrapper::~EnvWrapper() {} +EnvWrapper::~EnvWrapper() = default; Status EnvWrapper::PrepareOptions(const ConfigOptions& options) { target_.Prepare(); diff --git a/env/env.cc b/env/env.cc index 937be43c0..8ad828a83 100644 --- a/env/env.cc +++ b/env/env.cc @@ -359,7 +359,7 @@ class LegacyFileSystemWrapper : public FileSystem { public: // Initialize an EnvWrapper that delegates all calls to *t explicit LegacyFileSystemWrapper(Env* t) : target_(t) {} - ~LegacyFileSystemWrapper() override {} + ~LegacyFileSystemWrapper() override = default; static const char* kClassName() { return "LegacyFileSystem"; } const char* Name() const override { return kClassName(); } @@ -624,7 +624,7 @@ Env::Env(const std::shared_ptr& fs, const std::shared_ptr& clock) : thread_status_updater_(nullptr), file_system_(fs), system_clock_(clock) {} -Env::~Env() {} +Env::~Env() = default; Status Env::NewLogger(const std::string& fname, std::shared_ptr* result) { @@ -797,7 +797,7 @@ std::string Env::GenerateUniqueId() { // Use 36 character format of RFC 4122 result.resize(36U); - char* buf = &result[0]; + char* buf = result.data(); PutBaseChars<16>(&buf, 8, upper >> 32, /*!uppercase*/ false); *(buf++) = '-'; PutBaseChars<16>(&buf, 4, upper >> 16, /*!uppercase*/ false); @@ -817,15 +817,15 @@ std::string Env::GenerateUniqueId() { return result; } -SequentialFile::~SequentialFile() {} +SequentialFile::~SequentialFile() = default; -RandomAccessFile::~RandomAccessFile() {} +RandomAccessFile::~RandomAccessFile() = default; -WritableFile::~WritableFile() {} +WritableFile::~WritableFile() = default; -MemoryMappedFileBuffer::~MemoryMappedFileBuffer() {} +MemoryMappedFileBuffer::~MemoryMappedFileBuffer() = default; -Logger::~Logger() {} +Logger::~Logger() = default; Status Logger::Close() { if (!closed_) { @@ -838,7 +838,7 @@ Status Logger::Close() { Status Logger::CloseImpl() { return Status::NotSupported(); } -FileLock::~FileLock() {} +FileLock::~FileLock() = default; void LogFlush(Logger* info_log) { if (info_log) { @@ -1229,4 +1229,9 @@ Status SystemClock::CreateFromString(const ConfigOptions& config_options, return LoadSharedObject(config_options, value, result); } } + +bool SystemClock::TimedWait(port::CondVar* cv, + std::chrono::microseconds deadline) { + return cv->TimedWait(deadline.count()); +} } // namespace ROCKSDB_NAMESPACE diff --git a/env/env_basic_test.cc b/env/env_basic_test.cc index 11b07509c..6a3b0390a 100644 --- a/env/env_basic_test.cc +++ b/env/env_basic_test.cc @@ -57,10 +57,10 @@ static Env* GetTestEnv() { static std::shared_ptr env_guard; static Env* custom_env = nullptr; if (custom_env == nullptr) { - const char* uri = getenv("TEST_ENV_URI"); - if (uri != nullptr) { - EXPECT_OK(Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env, - &env_guard)); + const char* env_uri = getenv("TEST_ENV_URI"); + if (env_uri != nullptr) { + EXPECT_OK(Env::CreateFromUri(ConfigOptions(), env_uri, /*fs_uri=*/"", + &custom_env, &env_guard)); } } EXPECT_NE(custom_env, nullptr); @@ -71,10 +71,10 @@ static Env* GetTestFS() { static std::shared_ptr fs_env_guard; static Env* fs_env = nullptr; if (fs_env == nullptr) { - const char* uri = getenv("TEST_FS_URI"); - if (uri != nullptr) { - EXPECT_OK( - Env::CreateFromUri(ConfigOptions(), uri, "", &fs_env, &fs_env_guard)); + const char* fs_uri = getenv("TEST_FS_URI"); + if (fs_uri != nullptr) { + EXPECT_OK(Env::CreateFromUri(ConfigOptions(), /*env_uri=*/"", fs_uri, + &fs_env, &fs_env_guard)); } } EXPECT_NE(fs_env, nullptr); @@ -342,7 +342,7 @@ TEST_P(EnvMoreTestWithParam, GetChildren) { ASSERT_OK(env_->GetChildrenFileAttributes(test_dir_, &childAttr)); ASSERT_EQ(3U, children.size()); ASSERT_EQ(3U, childAttr.size()); - for (auto each : children) { + for (const auto& each : children) { env_->DeleteDir(test_dir_ + "/" + each).PermitUncheckedError(); } // necessary for default POSIX env diff --git a/env/env_chroot.cc b/env/env_chroot.cc index 5ff32a7e4..93dd7acd1 100644 --- a/env/env_chroot.cc +++ b/env/env_chroot.cc @@ -7,10 +7,11 @@ #include "env/env_chroot.h" -#include // errno -#include // realpath, free #include // geteuid +#include // errno +#include // realpath, free + #include "env/composite_env_wrapper.h" #include "env/fs_remap.h" #include "rocksdb/utilities/options_type.h" diff --git a/env/env_encryption.cc b/env/env_encryption.cc index 7b2a531c4..dd6ca54ea 100644 --- a/env/env_encryption.cc +++ b/env/env_encryption.cc @@ -827,7 +827,7 @@ Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data, AllocateScratch(scratch); // Encrypt individual blocks. - while (1) { + while (true) { char* block = data; size_t n = std::min(dataSize, blockSize - blockOffset); if (n != blockSize) { @@ -871,7 +871,7 @@ Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char* data, AllocateScratch(scratch); // Decrypt individual blocks. - while (1) { + while (true) { char* block = data; size_t n = std::min(dataSize, blockSize - blockOffset); if (n != blockSize) { diff --git a/env/env_posix.cc b/env/env_posix.cc index ae2f90360..9a02a7d11 100644 --- a/env/env_posix.cc +++ b/env/env_posix.cc @@ -14,19 +14,21 @@ #ifndef ROCKSDB_NO_DYNAMIC_EXTENSION #include #endif -#include #include +#include + #if defined(ROCKSDB_IOURING_PRESENT) #include #endif #include -#include -#include -#include -#include #include #include + +#include +#include +#include +#include #if defined(OS_LINUX) || defined(OS_SOLARIS) || defined(OS_ANDROID) #include #endif @@ -36,10 +38,10 @@ #if defined(ROCKSDB_IOURING_PRESENT) #include #endif -#include #include #include +#include // Get nano time includes #if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD) #elif defined(__MACH__) @@ -199,7 +201,7 @@ class PosixClock : public SystemClock { std::string dummy; dummy.reserve(maxsize); dummy.resize(maxsize); - char* p = &dummy[0]; + char* p = dummy.data(); port::LocalTimeR(&seconds, &t); snprintf(p, maxsize, "%04d/%02d/%02d-%02d:%02d:%02d ", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec); diff --git a/env/env_test.cc b/env/env_test.cc index 4462b95b8..4cf3c988d 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -26,13 +26,14 @@ #ifdef OS_LINUX #include #include -#include #include #include + +#include #endif #ifdef ROCKSDB_FALLOCATE_PRESENT -#include +#include #endif #include "db/db_impl/db_impl.h" @@ -2906,6 +2907,13 @@ TEST_F(CreateEnvTest, CreateEncryptedFileSystem) { std::string base_opts = std::string("provider=1://test; id=") + EncryptedFileSystem::kClassName(); + // Rewrite the default FileSystem URI if the "TEST_FS_URI" environment + // variable is set. This is useful to test customer encryption plugins. + const char* uri = getenv("TEST_FS_URI"); + if (uri != nullptr) { + base_opts = uri; + } + // The EncryptedFileSystem requires a "provider" option. ASSERT_NOK(FileSystem::CreateFromString( config_options_, EncryptedFileSystem::kClassName(), &fs)); @@ -2948,7 +2956,7 @@ struct NoDuplicateMiniStressTest { NoDuplicateMiniStressTest() { env = Env::Default(); } - virtual ~NoDuplicateMiniStressTest() {} + virtual ~NoDuplicateMiniStressTest() = default; void Run() { std::array threads; @@ -3054,7 +3062,7 @@ TEST_F(EnvTest, PortGenerateRfcUuid) { VerifyRfcUuids(t.ids); } -// Test the atomic, linear generation of GenerateRawUuid +// Test the atomic, linear generation of GenerateRawUniqueId TEST_F(EnvTest, GenerateRawUniqueId) { struct MyStressTest : public NoDuplicateMiniStressTest { diff --git a/env/file_system.cc b/env/file_system.cc index 71fb4d5bc..e01ec12c9 100644 --- a/env/file_system.cc +++ b/env/file_system.cc @@ -22,9 +22,9 @@ namespace ROCKSDB_NAMESPACE { -FileSystem::FileSystem() {} +FileSystem::FileSystem() = default; -FileSystem::~FileSystem() {} +FileSystem::~FileSystem() = default; static int RegisterBuiltinFileSystems(ObjectLibrary& library, const std::string& /*arg*/) { diff --git a/env/fs_posix.cc b/env/fs_posix.cc index dd2f74935..5f403e61e 100644 --- a/env/fs_posix.cc +++ b/env/fs_posix.cc @@ -13,16 +13,17 @@ #ifndef ROCKSDB_NO_DYNAMIC_EXTENSION #include #endif -#include #include #include -#include -#include -#include -#include #include #include #include + +#include +#include +#include +#include +#include #if defined(OS_LINUX) || defined(OS_SOLARIS) || defined(OS_ANDROID) #include #include @@ -30,9 +31,9 @@ #include #include #include -#include #include +#include // Get nano time includes #if defined(OS_LINUX) || defined(OS_FREEBSD) #elif defined(__MACH__) @@ -143,7 +144,7 @@ class PosixFileSystem : public FileSystem { const char* Name() const override { return kClassName(); } const char* NickName() const override { return kDefaultName(); } - ~PosixFileSystem() override {} + ~PosixFileSystem() override = default; bool IsInstanceOf(const std::string& name) const override { if (name == "posix") { return true; diff --git a/env/io_posix.cc b/env/io_posix.cc index 0ec0e9c83..7fde1d020 100644 --- a/env/io_posix.cc +++ b/env/io_posix.cc @@ -10,23 +10,24 @@ #ifdef ROCKSDB_LIB_IO_POSIX #include "env/io_posix.h" -#include #include #include +#include #if defined(OS_LINUX) #include #ifndef FALLOC_FL_KEEP_SIZE #include #endif #endif -#include -#include -#include #include #include #include #include + +#include +#include +#include #ifdef OS_LINUX #include #include @@ -437,7 +438,7 @@ void LogicalBlockSizeCache::UnrefAndTryRemoveCachedLogicalBlockSize( size_t LogicalBlockSizeCache::GetLogicalBlockSize(const std::string& fname, int fd) { - std::string dir = fname.substr(0, fname.find_last_of("/")); + std::string dir = fname.substr(0, fname.find_last_of('/')); if (dir.empty()) { dir = "/"; } @@ -654,7 +655,9 @@ IOStatus PosixRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs, size_t this_reqs = (num_reqs - reqs_off) + incomplete_rq_list.size(); // If requests exceed depth, split it into batches - if (this_reqs > kIoUringDepth) this_reqs = kIoUringDepth; + if (this_reqs > kIoUringDepth) { + this_reqs = kIoUringDepth; + } assert(incomplete_rq_list.size() <= this_reqs); for (size_t i = 0; i < this_reqs; i++) { diff --git a/env/io_posix.h b/env/io_posix.h index f129668ea..8c51ba645 100644 --- a/env/io_posix.h +++ b/env/io_posix.h @@ -29,7 +29,8 @@ // For non linux platform, the following macros are used only as place // holder. -#if !(defined OS_LINUX) && !(defined CYGWIN) && !(defined OS_AIX) +#if !(defined OS_LINUX) && !(defined OS_FREEBSD) && !(defined CYGWIN) && \ + !(defined OS_AIX) #define POSIX_FADV_NORMAL 0 /* [MC1] no further special treatment */ #define POSIX_FADV_RANDOM 1 /* [MC1] expect random page refs */ #define POSIX_FADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */ diff --git a/env/mock_env.cc b/env/mock_env.cc index c232af61e..e206593a2 100644 --- a/env/mock_env.cc +++ b/env/mock_env.cc @@ -483,7 +483,7 @@ class TestMemLogger : public Logger { options_(options), dbg_(dbg), flush_pending_(false) {} - ~TestMemLogger() override {} + ~TestMemLogger() override = default; void Flush() override { if (flush_pending_) { diff --git a/env/mock_env_test.cc b/env/mock_env_test.cc index be174bd73..23c4baa12 100644 --- a/env/mock_env_test.cc +++ b/env/mock_env_test.cc @@ -37,28 +37,28 @@ TEST_F(MockEnvTest, Corrupt) { Slice result; std::unique_ptr rand_file; ASSERT_OK(env_->NewRandomAccessFile(kFileName, &rand_file, soptions_)); - ASSERT_OK(rand_file->Read(0, kGood.size(), &result, &(scratch[0]))); + ASSERT_OK(rand_file->Read(0, kGood.size(), &result, scratch.data())); ASSERT_EQ(result.compare(kGood), 0); // Sync + corrupt => no change ASSERT_OK(writable_file->Fsync()); ASSERT_OK(dynamic_cast(env_)->CorruptBuffer(kFileName)); result.clear(); - ASSERT_OK(rand_file->Read(0, kGood.size(), &result, &(scratch[0]))); + ASSERT_OK(rand_file->Read(0, kGood.size(), &result, scratch.data())); ASSERT_EQ(result.compare(kGood), 0); // Add new data and corrupt it ASSERT_OK(writable_file->Append(kCorrupted)); ASSERT_TRUE(writable_file->GetFileSize() == kGood.size() + kCorrupted.size()); result.clear(); - ASSERT_OK( - rand_file->Read(kGood.size(), kCorrupted.size(), &result, &(scratch[0]))); + ASSERT_OK(rand_file->Read(kGood.size(), kCorrupted.size(), &result, + scratch.data())); ASSERT_EQ(result.compare(kCorrupted), 0); // Corrupted ASSERT_OK(dynamic_cast(env_)->CorruptBuffer(kFileName)); result.clear(); - ASSERT_OK( - rand_file->Read(kGood.size(), kCorrupted.size(), &result, &(scratch[0]))); + ASSERT_OK(rand_file->Read(kGood.size(), kCorrupted.size(), &result, + scratch.data())); ASSERT_NE(result.compare(kCorrupted), 0); } diff --git a/env/unique_id_gen.cc b/env/unique_id_gen.cc index 7d221d374..8d9db8695 100644 --- a/env/unique_id_gen.cc +++ b/env/unique_id_gen.cc @@ -21,6 +21,7 @@ #ifdef __SSE4_2__ #ifdef _WIN32 #include +#define _rdtsc() __rdtsc() #else #include #endif diff --git a/examples/compact_files_example.cc b/examples/compact_files_example.cc index 1ecf8c794..544adf8ae 100644 --- a/examples/compact_files_example.cc +++ b/examples/compact_files_example.cc @@ -144,6 +144,8 @@ int main() { options.create_if_missing = true; // Disable RocksDB background compaction. options.compaction_style = ROCKSDB_NAMESPACE::kCompactionStyleNone; + // Small write buffer size for generating more sst files in level 0. + options.write_buffer_size = 4 << 20; // Small slowdown and stop trigger for experimental purpose. options.level0_slowdown_writes_trigger = 3; options.level0_stop_writes_trigger = 5; diff --git a/file/delete_scheduler.cc b/file/delete_scheduler.cc index 78ea6f7fe..9e7dd3d60 100644 --- a/file/delete_scheduler.cc +++ b/file/delete_scheduler.cc @@ -177,7 +177,7 @@ Status DeleteScheduler::CleanupDirectory(Env* env, SstFileManagerImpl* sfm, Status DeleteScheduler::MarkAsTrash(const std::string& file_path, std::string* trash_file) { // Sanity check of the path - size_t idx = file_path.rfind("/"); + size_t idx = file_path.rfind('/'); if (idx == std::string::npos || idx == file_path.size() - 1) { return Status::InvalidArgument("file_path is corrupted"); } diff --git a/file/delete_scheduler_test.cc b/file/delete_scheduler_test.cc index 62de5d27c..46e834879 100644 --- a/file/delete_scheduler_test.cc +++ b/file/delete_scheduler_test.cc @@ -40,7 +40,7 @@ class DeleteSchedulerTest : public testing::Test { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({}); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); for (const auto& dummy_files_dir : dummy_files_dirs_) { - DestroyDir(env_, dummy_files_dir); + EXPECT_OK(DestroyDir(env_, dummy_files_dir)); } } @@ -82,11 +82,11 @@ class DeleteSchedulerTest : public testing::Test { std::string file_path = dummy_files_dirs_[dummy_files_dirs_idx] + "/" + file_name; std::unique_ptr f; - env_->NewWritableFile(file_path, &f, EnvOptions()); + EXPECT_OK(env_->NewWritableFile(file_path, &f, EnvOptions())); std::string data(size, 'A'); EXPECT_OK(f->Append(data)); EXPECT_OK(f->Close()); - sst_file_mgr_->OnAddFile(file_path); + EXPECT_OK(sst_file_mgr_->OnAddFile(file_path)); return file_path; } diff --git a/file/file_prefetch_buffer.cc b/file/file_prefetch_buffer.cc index 59fbf1285..2bd8c4a85 100644 --- a/file/file_prefetch_buffer.cc +++ b/file/file_prefetch_buffer.cc @@ -81,13 +81,12 @@ void FilePrefetchBuffer::CalculateOffsetAndLen(size_t alignment, Status FilePrefetchBuffer::Read(const IOOptions& opts, RandomAccessFileReader* reader, - Env::IOPriority rate_limiter_priority, uint64_t read_len, uint64_t chunk_len, - uint64_t rounddown_start, uint32_t index) { + uint64_t start_offset, uint32_t index) { Slice result; - Status s = reader->Read(opts, rounddown_start + chunk_len, read_len, &result, + Status s = reader->Read(opts, start_offset + chunk_len, read_len, &result, bufs_[index].buffer_.BufferStart() + chunk_len, - /*aligned_buf=*/nullptr, rate_limiter_priority); + /*aligned_buf=*/nullptr); #ifndef NDEBUG if (result.size() < read_len) { // Fake an IO error to force db_stress fault injection to ignore @@ -99,16 +98,19 @@ Status FilePrefetchBuffer::Read(const IOOptions& opts, return s; } + if (usage_ == FilePrefetchBufferUsage::kUserScanPrefetch) { + RecordTick(stats_, PREFETCH_BYTES, read_len); + } // Update the buffer offset and size. - bufs_[index].offset_ = rounddown_start; + bufs_[index].offset_ = start_offset; bufs_[index].buffer_.Size(static_cast(chunk_len) + result.size()); return s; } Status FilePrefetchBuffer::ReadAsync(const IOOptions& opts, RandomAccessFileReader* reader, - uint64_t read_len, - uint64_t rounddown_start, uint32_t index) { + uint64_t read_len, uint64_t start_offset, + uint32_t index) { TEST_SYNC_POINT("FilePrefetchBuffer::ReadAsync"); // callback for async read request. auto fp = std::bind(&FilePrefetchBuffer::PrefetchAsyncCallback, this, @@ -116,7 +118,7 @@ Status FilePrefetchBuffer::ReadAsync(const IOOptions& opts, FSReadRequest req; Slice result; req.len = read_len; - req.offset = rounddown_start; + req.offset = start_offset; req.result = result; req.scratch = bufs_[index].buffer_.BufferStart(); bufs_[index].async_req_len_ = req.len; @@ -127,6 +129,7 @@ Status FilePrefetchBuffer::ReadAsync(const IOOptions& opts, /*aligned_buf=*/nullptr); req.status.PermitUncheckedError(); if (s.ok()) { + RecordTick(stats_, PREFETCH_BYTES, read_len); bufs_[index].async_read_in_progress_ = true; } return s; @@ -134,8 +137,7 @@ Status FilePrefetchBuffer::ReadAsync(const IOOptions& opts, Status FilePrefetchBuffer::Prefetch(const IOOptions& opts, RandomAccessFileReader* reader, - uint64_t offset, size_t n, - Env::IOPriority rate_limiter_priority) { + uint64_t offset, size_t n) { if (!enable_ || reader == nullptr) { return Status::OK(); } @@ -148,20 +150,18 @@ Status FilePrefetchBuffer::Prefetch(const IOOptions& opts, } size_t alignment = reader->file()->GetRequiredBufferAlignment(); - size_t offset_ = static_cast(offset); - uint64_t rounddown_offset = Rounddown(offset_, alignment); - uint64_t roundup_end = Roundup(offset_ + n, alignment); - uint64_t roundup_len = roundup_end - rounddown_offset; - assert(roundup_len >= alignment); - assert(roundup_len % alignment == 0); - - uint64_t chunk_len = 0; - CalculateOffsetAndLen(alignment, offset, roundup_len, curr_, - true /*refit_tail*/, chunk_len); - size_t read_len = static_cast(roundup_len - chunk_len); - - Status s = Read(opts, reader, rate_limiter_priority, read_len, chunk_len, - rounddown_offset, curr_); + uint64_t rounddown_offset = offset, roundup_end = 0, chunk_len = 0; + size_t read_len = 0; + + ReadAheadSizeTuning(/*read_curr_block=*/true, /*refit_tail=*/true, + rounddown_offset, curr_, alignment, 0, n, + rounddown_offset, roundup_end, read_len, chunk_len); + + Status s; + if (read_len > 0) { + s = Read(opts, reader, read_len, chunk_len, rounddown_offset, curr_); + } + if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && s.ok()) { RecordInHistogram(stats_, TABLE_OPEN_PREFETCH_TAIL_READ_BYTES, read_len); } @@ -195,7 +195,7 @@ void FilePrefetchBuffer::CopyDataToBuffer(uint32_t src, uint64_t& offset, // length > 0 indicates it has consumed all data from the src buffer and it // still needs to read more other buffer. if (length > 0) { - bufs_[src].buffer_.Clear(); + bufs_[src].ClearBuffer(); } } @@ -265,28 +265,35 @@ void FilePrefetchBuffer::AbortAllIOs() { // Clear the buffers if it contains outdated data. Outdated data can be // because previous sequential reads were read from the cache instead of these // buffer. -void FilePrefetchBuffer::UpdateBuffersIfNeeded(uint64_t offset) { +void FilePrefetchBuffer::UpdateBuffersIfNeeded(uint64_t offset, size_t length) { uint32_t second = curr_ ^ 1; + if (IsBufferOutdated(offset, curr_)) { - bufs_[curr_].buffer_.Clear(); + bufs_[curr_].ClearBuffer(); } if (IsBufferOutdated(offset, second)) { - bufs_[second].buffer_.Clear(); + bufs_[second].ClearBuffer(); } { - // In case buffers do not align, reset second buffer. This can happen in - // case readahead_size is set. + // In case buffers do not align, reset second buffer if requested data needs + // to be read in second buffer. if (!bufs_[second].async_read_in_progress_ && !bufs_[curr_].async_read_in_progress_) { if (DoesBufferContainData(curr_)) { if (bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize() != bufs_[second].offset_) { - bufs_[second].buffer_.Clear(); + if (DoesBufferContainData(second) && + IsOffsetInBuffer(offset, curr_) && + (offset + length > + bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize())) { + bufs_[second].ClearBuffer(); + } } } else { - if (!IsOffsetInBuffer(offset, second)) { - bufs_[second].buffer_.Clear(); + if (DoesBufferContainData(second) && + !IsOffsetInBuffer(offset, second)) { + bufs_[second].ClearBuffer(); } } } @@ -307,7 +314,8 @@ void FilePrefetchBuffer::UpdateBuffersIfNeeded(uint64_t offset) { } } -void FilePrefetchBuffer::PollAndUpdateBuffersIfNeeded(uint64_t offset) { +void FilePrefetchBuffer::PollAndUpdateBuffersIfNeeded(uint64_t offset, + size_t length) { if (bufs_[curr_].async_read_in_progress_ && fs_ != nullptr) { if (bufs_[curr_].io_handle_ != nullptr) { // Wait for prefetch data to complete. @@ -323,13 +331,92 @@ void FilePrefetchBuffer::PollAndUpdateBuffersIfNeeded(uint64_t offset) { // completed. DestroyAndClearIOHandle(curr_); } - UpdateBuffersIfNeeded(offset); + UpdateBuffersIfNeeded(offset, length); +} + +// ReadAheadSizeTuning API calls readaheadsize_cb_ +// (BlockBasedTableIterator::BlockCacheLookupForReadAheadSize) to lookup in the +// cache and tune the start and end offsets based on cache hits/misses. +// +// Arguments - +// read_curr_block : True if this call was due to miss in the cache and +// FilePrefetchBuffer wants to read that block +// synchronously. +// False if current call is to prefetch additional data in +// extra buffers through ReadAsync API. +// prev_buf_end_offset : End offset of the previous buffer. It's used in case +// of ReadAsync to make sure it doesn't read anything from +// previous buffer which is already prefetched. +void FilePrefetchBuffer::ReadAheadSizeTuning( + bool read_curr_block, bool refit_tail, uint64_t prev_buf_end_offset, + uint32_t index, size_t alignment, size_t length, size_t readahead_size, + uint64_t& start_offset, uint64_t& end_offset, size_t& read_len, + uint64_t& chunk_len) { + uint64_t updated_start_offset = Rounddown(start_offset, alignment); + uint64_t updated_end_offset = + Roundup(start_offset + length + readahead_size, alignment); + uint64_t initial_end_offset = updated_end_offset; + uint64_t initial_start_offset = updated_start_offset; + + // Callback to tune the start and end offsets. + if (readaheadsize_cb_ != nullptr && readahead_size > 0) { + readaheadsize_cb_(read_curr_block, updated_start_offset, + updated_end_offset); + } + + // read_len will be 0 and there is nothing to read/prefetch. + if (updated_start_offset == updated_end_offset) { + UpdateReadAheadTrimmedStat((initial_end_offset - initial_start_offset), + (updated_end_offset - updated_start_offset)); + return; + } + + assert(updated_start_offset < updated_end_offset); + + if (!read_curr_block) { + // Handle the case when callback added block handles which are already + // prefetched and nothing new needs to be prefetched. In that case end + // offset updated by callback will be less than prev_buf_end_offset which + // means data has been already prefetched. + if (updated_end_offset <= prev_buf_end_offset) { + start_offset = end_offset = prev_buf_end_offset; + UpdateReadAheadTrimmedStat((initial_end_offset - initial_start_offset), + (end_offset - start_offset)); + return; + } + } + + // Realign if start and end offsets are not aligned after tuning. + start_offset = Rounddown(updated_start_offset, alignment); + end_offset = Roundup(updated_end_offset, alignment); + + if (!read_curr_block && start_offset < prev_buf_end_offset) { + // Previous buffer already contains the data till prev_buf_end_offset + // because of alignment. Update the start offset after that to avoid + // prefetching it again. + start_offset = prev_buf_end_offset; + } + + uint64_t roundup_len = end_offset - start_offset; + + CalculateOffsetAndLen(alignment, start_offset, roundup_len, index, refit_tail, + chunk_len); + assert(roundup_len >= chunk_len); + + // Update the buffer offset. + bufs_[index].offset_ = start_offset; + // Update the initial end offset of this buffer which will be the starting + // offset of next prefetch. + bufs_[index].initial_end_offset_ = initial_end_offset; + read_len = static_cast(roundup_len - chunk_len); + + UpdateReadAheadTrimmedStat((initial_end_offset - initial_start_offset), + (end_offset - start_offset)); } Status FilePrefetchBuffer::HandleOverlappingData( const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, - size_t length, size_t readahead_size, - Env::IOPriority /*rate_limiter_priority*/, bool& copy_to_third_buffer, + size_t length, size_t readahead_size, bool& copy_to_third_buffer, uint64_t& tmp_offset, size_t& tmp_length) { Status s; size_t alignment = reader->file()->GetRequiredBufferAlignment(); @@ -340,7 +427,7 @@ Status FilePrefetchBuffer::HandleOverlappingData( // by Seek, but the next access is at another offset. if (bufs_[curr_].async_read_in_progress_ && IsOffsetInBufferWithAsyncProgress(offset, curr_)) { - PollAndUpdateBuffersIfNeeded(offset); + PollAndUpdateBuffersIfNeeded(offset, length); } second = curr_ ^ 1; @@ -354,7 +441,7 @@ Status FilePrefetchBuffer::HandleOverlappingData( (bufs_[second].async_read_in_progress_ || DoesBufferContainData(second)))) { // Allocate new buffer to third buffer; - bufs_[2].buffer_.Clear(); + bufs_[2].ClearBuffer(); bufs_[2].buffer_.Alignment(alignment); bufs_[2].buffer_.AllocateNewBuffer(length); bufs_[2].offset_ = offset; @@ -363,28 +450,28 @@ Status FilePrefetchBuffer::HandleOverlappingData( CopyDataToBuffer(curr_, tmp_offset, tmp_length); // Call async prefetching on curr_ since data has been consumed in curr_ - // only if data lies within second buffer. + // only if requested data lies within second buffer. size_t second_size = bufs_[second].async_read_in_progress_ ? bufs_[second].async_req_len_ : bufs_[second].buffer_.CurrentSize(); + uint64_t start_offset = bufs_[second].initial_end_offset_; + // Second buffer might be out of bound if first buffer already prefetched + // that data. if (tmp_offset + tmp_length <= bufs_[second].offset_ + second_size) { - uint64_t rounddown_start = bufs_[second].offset_ + second_size; - uint64_t roundup_end = - Roundup(rounddown_start + readahead_size, alignment); - uint64_t roundup_len = roundup_end - rounddown_start; - uint64_t chunk_len = 0; - CalculateOffsetAndLen(alignment, rounddown_start, roundup_len, curr_, - false, chunk_len); - assert(chunk_len == 0); - assert(roundup_len >= chunk_len); - - bufs_[curr_].offset_ = rounddown_start; - uint64_t read_len = static_cast(roundup_len - chunk_len); - s = ReadAsync(opts, reader, read_len, rounddown_start, curr_); - if (!s.ok()) { - DestroyAndClearIOHandle(curr_); - bufs_[curr_].buffer_.Clear(); - return s; + size_t read_len = 0; + uint64_t end_offset = start_offset, chunk_len = 0; + + ReadAheadSizeTuning(/*read_curr_block=*/false, /*refit_tail=*/false, + bufs_[second].offset_ + second_size, curr_, alignment, + /*length=*/0, readahead_size, start_offset, + end_offset, read_len, chunk_len); + if (read_len > 0) { + s = ReadAsync(opts, reader, read_len, start_offset, curr_); + if (!s.ok()) { + DestroyAndClearIOHandle(curr_); + bufs_[curr_].ClearBuffer(); + return s; + } } } curr_ = curr_ ^ 1; @@ -412,10 +499,11 @@ Status FilePrefetchBuffer::HandleOverlappingData( // curr_, send async request on curr_, wait for poll to fill second // buffer (if any), and copy remaining data from second buffer to third // buffer. -Status FilePrefetchBuffer::PrefetchAsyncInternal( - const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, - size_t length, size_t readahead_size, Env::IOPriority rate_limiter_priority, - bool& copy_to_third_buffer) { +Status FilePrefetchBuffer::PrefetchAsyncInternal(const IOOptions& opts, + RandomAccessFileReader* reader, + uint64_t offset, size_t length, + size_t readahead_size, + bool& copy_to_third_buffer) { if (!enable_) { return Status::OK(); } @@ -426,13 +514,14 @@ Status FilePrefetchBuffer::PrefetchAsyncInternal( Status s; uint64_t tmp_offset = offset; size_t tmp_length = length; + size_t original_length = length; // 1. Abort IO and swap buffers if needed to point curr_ to first buffer with // data. if (!explicit_prefetch_submitted_) { AbortIOIfNeeded(offset); } - UpdateBuffersIfNeeded(offset); + UpdateBuffersIfNeeded(offset, length); // 2. Handle overlapping data over two buffers. If data is overlapping then // during this call: @@ -442,8 +531,7 @@ Status FilePrefetchBuffer::PrefetchAsyncInternal( // - switch buffers and curr_ now points to second buffer to copy remaining // data. s = HandleOverlappingData(opts, reader, offset, length, readahead_size, - rate_limiter_priority, copy_to_third_buffer, - tmp_offset, tmp_length); + copy_to_third_buffer, tmp_offset, tmp_length); if (!s.ok()) { return s; } @@ -455,14 +543,15 @@ Status FilePrefetchBuffer::PrefetchAsyncInternal( if (!bufs_[curr_].async_read_in_progress_ && DoesBufferContainData(curr_) && IsDataBlockInBuffer(offset, length, curr_)) { // Whole data is in curr_. - UpdateBuffersIfNeeded(offset); + UpdateBuffersIfNeeded(offset, length); if (!IsSecondBuffEligibleForPrefetching()) { + UpdateStats(/*found_in_buffer=*/true, original_length); return s; } } else { // After poll request, curr_ might be empty because of IOError in // callback while reading or may contain required data. - PollAndUpdateBuffersIfNeeded(offset); + PollAndUpdateBuffersIfNeeded(offset, length); } if (copy_to_third_buffer) { @@ -485,6 +574,7 @@ Status FilePrefetchBuffer::PrefetchAsyncInternal( return s; } if (!IsSecondBuffEligibleForPrefetching()) { + UpdateStats(/*found_in_buffer=*/true, original_length); return s; } } @@ -506,7 +596,7 @@ Status FilePrefetchBuffer::PrefetchAsyncInternal( } } DestroyAndClearIOHandle(second); - bufs_[second].buffer_.Clear(); + bufs_[second].ClearBuffer(); } // 5. Data is overlapping i.e. some of the data has been copied to third @@ -521,68 +611,57 @@ Status FilePrefetchBuffer::PrefetchAsyncInternal( // and sync prefetching and copy the remaining data to third buffer in the // end. if (length == 0) { + UpdateStats(/*found_in_buffer=*/true, original_length); return s; } } // 6. Go for ReadAsync and Read (if needed). - size_t prefetch_size = length + readahead_size; - size_t _offset = static_cast(offset); - - // offset and size alignment for curr_ buffer with synchronous prefetching - uint64_t rounddown_start1 = Rounddown(_offset, alignment); - uint64_t roundup_end1 = Roundup(_offset + prefetch_size, alignment); - uint64_t roundup_len1 = roundup_end1 - rounddown_start1; - assert(roundup_len1 >= alignment); - assert(roundup_len1 % alignment == 0); - uint64_t chunk_len1 = 0; - uint64_t read_len1 = 0; - assert(!bufs_[second].async_read_in_progress_ && !DoesBufferContainData(second)); + // offset and size alignment for curr_ buffer with synchronous prefetching + uint64_t start_offset1 = offset, end_offset1 = 0, chunk_len1 = 0; + size_t read_len1 = 0; + // For length == 0, skip the synchronous prefetching. read_len1 will be 0. if (length > 0) { - CalculateOffsetAndLen(alignment, offset, roundup_len1, curr_, - false /*refit_tail*/, chunk_len1); - assert(roundup_len1 >= chunk_len1); - read_len1 = static_cast(roundup_len1 - chunk_len1); + ReadAheadSizeTuning(/*read_curr_block=*/true, /*refit_tail=*/false, + start_offset1, curr_, alignment, length, readahead_size, + start_offset1, end_offset1, read_len1, chunk_len1); + UpdateStats(/*found_in_buffer=*/false, + /*length_found=*/original_length - length); + } else { + end_offset1 = bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize(); + UpdateStats(/*found_in_buffer=*/true, original_length); } - { - // offset and size alignment for second buffer for asynchronous - // prefetching - uint64_t rounddown_start2 = roundup_end1; - uint64_t roundup_end2 = - Roundup(rounddown_start2 + readahead_size, alignment); - // For length == 0, do the asynchronous prefetching in second instead of - // synchronous prefetching in curr_. - if (length == 0) { - rounddown_start2 = - bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize(); - roundup_end2 = Roundup(rounddown_start2 + prefetch_size, alignment); - } - - uint64_t roundup_len2 = roundup_end2 - rounddown_start2; - uint64_t chunk_len2 = 0; - CalculateOffsetAndLen(alignment, rounddown_start2, roundup_len2, second, - false /*refit_tail*/, chunk_len2); - assert(chunk_len2 == 0); - // Update the buffer offset. - bufs_[second].offset_ = rounddown_start2; - assert(roundup_len2 >= chunk_len2); - uint64_t read_len2 = static_cast(roundup_len2 - chunk_len2); - s = ReadAsync(opts, reader, read_len2, rounddown_start2, second); - if (!s.ok()) { - DestroyAndClearIOHandle(second); - bufs_[second].buffer_.Clear(); - return s; + // Prefetch in second buffer only if readahead_size > 0. + if (readahead_size > 0) { + // offset and size alignment for second buffer for asynchronous + // prefetching. + uint64_t start_offset2 = bufs_[curr_].initial_end_offset_; + + // Find updated readahead size after tuning + size_t read_len2 = 0; + uint64_t end_offset2 = start_offset2, chunk_len2 = 0; + ReadAheadSizeTuning(/*read_curr_block=*/false, /*refit_tail=*/false, + /*prev_buf_end_offset=*/end_offset1, second, + alignment, + /*length=*/0, readahead_size, start_offset2, + end_offset2, read_len2, chunk_len2); + if (read_len2 > 0) { + s = ReadAsync(opts, reader, read_len2, start_offset2, second); + if (!s.ok()) { + DestroyAndClearIOHandle(second); + bufs_[second].ClearBuffer(); + return s; + } } } if (read_len1 > 0) { - s = Read(opts, reader, rate_limiter_priority, read_len1, chunk_len1, - rounddown_start1, curr_); + s = Read(opts, reader, read_len1, chunk_len1, start_offset1, curr_); if (!s.ok()) { if (bufs_[second].io_handle_ != nullptr) { std::vector handles; @@ -594,11 +673,12 @@ Status FilePrefetchBuffer::PrefetchAsyncInternal( } } DestroyAndClearIOHandle(second); - bufs_[second].buffer_.Clear(); - bufs_[curr_].buffer_.Clear(); + bufs_[second].ClearBuffer(); + bufs_[curr_].ClearBuffer(); return s; } } + // Copy remaining requested bytes to third_buffer. if (copy_to_third_buffer && length > 0) { CopyDataToBuffer(curr_, offset, length); @@ -610,10 +690,9 @@ bool FilePrefetchBuffer::TryReadFromCache(const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, size_t n, Slice* result, Status* status, - Env::IOPriority rate_limiter_priority, bool for_compaction /* = false */) { bool ret = TryReadFromCacheUntracked(opts, reader, offset, n, result, status, - rate_limiter_priority, for_compaction); + for_compaction); if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && enable_) { if (ret) { RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_HIT); @@ -627,7 +706,7 @@ bool FilePrefetchBuffer::TryReadFromCache(const IOOptions& opts, bool FilePrefetchBuffer::TryReadFromCacheUntracked( const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, size_t n, Slice* result, Status* status, - Env::IOPriority rate_limiter_priority, bool for_compaction /* = false */) { + bool for_compaction /* = false */) { if (track_min_offset_ && offset < min_offset_read_) { min_offset_read_ = static_cast(offset); } @@ -647,9 +726,13 @@ bool FilePrefetchBuffer::TryReadFromCacheUntracked( assert(reader != nullptr); assert(max_readahead_size_ >= readahead_size_); if (for_compaction) { - s = Prefetch(opts, reader, offset, std::max(n, readahead_size_), - rate_limiter_priority); + s = Prefetch(opts, reader, offset, std::max(n, readahead_size_)); } else { + if (IsOffsetInBuffer(offset, curr_)) { + RecordTick(stats_, PREFETCH_BYTES_USEFUL, + bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize() - + offset); + } if (implicit_auto_readahead_) { if (!IsEligibleForPrefetch(offset, n)) { // Ignore status as Prefetch is not called. @@ -657,8 +740,7 @@ bool FilePrefetchBuffer::TryReadFromCacheUntracked( return false; } } - s = Prefetch(opts, reader, offset, n + readahead_size_, - rate_limiter_priority); + s = Prefetch(opts, reader, offset, n + readahead_size_); } if (!s.ok()) { if (status) { @@ -673,6 +755,9 @@ bool FilePrefetchBuffer::TryReadFromCacheUntracked( } else { return false; } + } else if (!for_compaction) { + RecordTick(stats_, PREFETCH_HITS); + RecordTick(stats_, PREFETCH_BYTES_USEFUL, n); } UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/); @@ -681,12 +766,12 @@ bool FilePrefetchBuffer::TryReadFromCacheUntracked( return true; } -bool FilePrefetchBuffer::TryReadFromCacheAsync( - const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, - size_t n, Slice* result, Status* status, - Env::IOPriority rate_limiter_priority) { - bool ret = TryReadFromCacheAsyncUntracked(opts, reader, offset, n, result, - status, rate_limiter_priority); +bool FilePrefetchBuffer::TryReadFromCacheAsync(const IOOptions& opts, + RandomAccessFileReader* reader, + uint64_t offset, size_t n, + Slice* result, Status* status) { + bool ret = + TryReadFromCacheAsyncUntracked(opts, reader, offset, n, result, status); if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && enable_) { if (ret) { RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_HIT); @@ -699,8 +784,7 @@ bool FilePrefetchBuffer::TryReadFromCacheAsync( bool FilePrefetchBuffer::TryReadFromCacheAsyncUntracked( const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, - size_t n, Slice* result, Status* status, - Env::IOPriority rate_limiter_priority) { + size_t n, Slice* result, Status* status) { if (track_min_offset_ && offset < min_offset_read_) { min_offset_read_ = static_cast(offset); } @@ -716,8 +800,8 @@ bool FilePrefetchBuffer::TryReadFromCacheAsyncUntracked( // Random offset called. So abort the IOs. if (prev_offset_ != offset) { AbortAllIOs(); - bufs_[curr_].buffer_.Clear(); - bufs_[curr_ ^ 1].buffer_.Clear(); + bufs_[curr_].ClearBuffer(); + bufs_[curr_ ^ 1].ClearBuffer(); explicit_prefetch_submitted_ = false; return false; } @@ -740,7 +824,9 @@ bool FilePrefetchBuffer::TryReadFromCacheAsyncUntracked( (bufs_[curr_].async_read_in_progress_ || offset + n > bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize())) { - if (readahead_size_ > 0) { + // In case readahead_size is trimmed (=0), we still want to poll the data + // submitted with explicit_prefetch_submitted_=true. + if (readahead_size_ > 0 || explicit_prefetch_submitted_) { Status s; assert(reader != nullptr); assert(max_readahead_size_ >= readahead_size_); @@ -752,10 +838,11 @@ bool FilePrefetchBuffer::TryReadFromCacheAsyncUntracked( return false; } } + // Prefetch n + readahead_size_/2 synchronously as remaining // readahead_size_/2 will be prefetched asynchronously. s = PrefetchAsyncInternal(opts, reader, offset, n, readahead_size_ / 2, - rate_limiter_priority, copy_to_third_buffer); + copy_to_third_buffer); explicit_prefetch_submitted_ = false; if (!s.ok()) { if (status) { @@ -770,6 +857,8 @@ bool FilePrefetchBuffer::TryReadFromCacheAsyncUntracked( } else { return false; } + } else { + UpdateStats(/*found_in_buffer=*/true, n); } UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/); @@ -829,10 +918,11 @@ Status FilePrefetchBuffer::PrefetchAsync(const IOOptions& opts, num_file_reads_ = 0; explicit_prefetch_submitted_ = false; bool is_eligible_for_prefetching = false; + if (readahead_size_ > 0 && (!implicit_auto_readahead_ || num_file_reads_ >= num_file_reads_for_auto_readahead_)) { - is_eligible_for_prefetching = true; + is_eligible_for_prefetching = true; } // 1. Cancel any pending async read to make code simpler as buffers can be out @@ -840,14 +930,19 @@ Status FilePrefetchBuffer::PrefetchAsync(const IOOptions& opts, AbortAllIOs(); // 2. Clear outdated data. - UpdateBuffersIfNeeded(offset); + UpdateBuffersIfNeeded(offset, n); uint32_t second = curr_ ^ 1; - // Since PrefetchAsync can be called on non sequential reads. So offset can - // be less than curr_ buffers' offset. In that case also it clears both - // buffers. - if (DoesBufferContainData(curr_) && !IsOffsetInBuffer(offset, curr_)) { - bufs_[curr_].buffer_.Clear(); - bufs_[second].buffer_.Clear(); + + // - Since PrefetchAsync can be called on non sequential reads. So offset can + // be less than curr_ buffers' offset. In that case it clears both + // buffers. + // - In case of tuning of readahead_size, on Reseek, we have to clear both + // buffers otherwise, we may end up with inconsistent BlockHandles in queue + // and data in buffer. + if (readaheadsize_cb_ != nullptr || + (DoesBufferContainData(curr_) && !IsOffsetInBuffer(offset, curr_))) { + bufs_[curr_].ClearBuffer(); + bufs_[second].ClearBuffer(); } UpdateReadPattern(offset, n, /*decrease_readaheadsize=*/false); @@ -859,6 +954,8 @@ Status FilePrefetchBuffer::PrefetchAsync(const IOOptions& opts, uint64_t offset_in_buffer = offset - bufs_[curr_].offset_; *result = Slice(bufs_[curr_].buffer_.BufferStart() + offset_in_buffer, n); data_found = true; + UpdateStats(/*found_in_buffer=*/true, n); + // Update num_file_reads_ as TryReadFromCacheAsync won't be called for // poll and update num_file_reads_ if data is found. num_file_reads_++; @@ -870,82 +967,78 @@ Status FilePrefetchBuffer::PrefetchAsync(const IOOptions& opts, } } else { // Partial data in curr_. - bufs_[curr_].buffer_.Clear(); + bufs_[curr_].ClearBuffer(); } - bufs_[second].buffer_.Clear(); + bufs_[second].ClearBuffer(); + + std::string msg; Status s; size_t alignment = reader->file()->GetRequiredBufferAlignment(); - size_t prefetch_size = is_eligible_for_prefetching ? readahead_size_ / 2 : 0; + size_t readahead_size = is_eligible_for_prefetching ? readahead_size_ / 2 : 0; size_t offset_to_read = static_cast(offset); - uint64_t rounddown_start1 = 0; - uint64_t roundup_end1 = 0; - uint64_t rounddown_start2 = 0; - uint64_t roundup_end2 = 0; - uint64_t chunk_len1 = 0; - uint64_t chunk_len2 = 0; - size_t read_len1 = 0; - size_t read_len2 = 0; + uint64_t start_offset1 = offset, end_offset1 = 0, start_offset2 = 0, + chunk_len1 = 0; + size_t read_len1 = 0, read_len2 = 0; // - If curr_ is empty. - // - Call async read for full data + prefetch_size on curr_. - // - Call async read for prefetch_size on second if eligible. + // - Call async read for full data + readahead_size on curr_. + // - Call async read for readahead_size on second if eligible. // - If curr_ is filled. - // - prefetch_size on second. + // - readahead_size on second. // Calculate length and offsets for reading. if (!DoesBufferContainData(curr_)) { - // Prefetch full data + prefetch_size in curr_. - rounddown_start1 = Rounddown(offset_to_read, alignment); - roundup_end1 = Roundup(offset_to_read + n + prefetch_size, alignment); - uint64_t roundup_len1 = roundup_end1 - rounddown_start1; - assert(roundup_len1 >= alignment); - assert(roundup_len1 % alignment == 0); - - CalculateOffsetAndLen(alignment, rounddown_start1, roundup_len1, curr_, - false, chunk_len1); - assert(chunk_len1 == 0); - assert(roundup_len1 >= chunk_len1); - read_len1 = static_cast(roundup_len1 - chunk_len1); - bufs_[curr_].offset_ = rounddown_start1; - } - - if (is_eligible_for_prefetching) { - if (DoesBufferContainData(curr_)) { - rounddown_start2 = - bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize(); + uint64_t roundup_len1; + // Prefetch full data + readahead_size in curr_. + if (is_eligible_for_prefetching || reader->use_direct_io()) { + ReadAheadSizeTuning(/*read_curr_block=*/true, /*refit_tail=*/false, + /*prev_buf_end_offset=*/start_offset1, curr_, + alignment, n, readahead_size, start_offset1, + end_offset1, read_len1, chunk_len1); } else { - rounddown_start2 = roundup_end1; + // No alignment or extra prefetching. + start_offset1 = offset_to_read; + end_offset1 = offset_to_read + n; + roundup_len1 = end_offset1 - start_offset1; + CalculateOffsetAndLen(alignment, start_offset1, roundup_len1, curr_, + false, chunk_len1); + assert(chunk_len1 == 0); + assert(roundup_len1 >= chunk_len1); + read_len1 = static_cast(roundup_len1); + bufs_[curr_].offset_ = start_offset1; } + } - roundup_end2 = Roundup(rounddown_start2 + prefetch_size, alignment); - uint64_t roundup_len2 = roundup_end2 - rounddown_start2; - - assert(roundup_len2 >= alignment); - CalculateOffsetAndLen(alignment, rounddown_start2, roundup_len2, second, - false, chunk_len2); - assert(chunk_len2 == 0); - assert(roundup_len2 >= chunk_len2); - read_len2 = static_cast(roundup_len2 - chunk_len2); - // Update the buffer offset. - bufs_[second].offset_ = rounddown_start2; + if (is_eligible_for_prefetching) { + start_offset2 = bufs_[curr_].initial_end_offset_; + // Second buffer might be out of bound if first buffer already prefetched + // that data. + + uint64_t end_offset2 = start_offset2, chunk_len2 = 0; + ReadAheadSizeTuning(/*read_curr_block=*/false, /*refit_tail=*/false, + /*prev_buf_end_offset=*/end_offset1, second, + alignment, + /*length=*/0, readahead_size, start_offset2, + end_offset2, read_len2, chunk_len2); } if (read_len1) { - s = ReadAsync(opts, reader, read_len1, rounddown_start1, curr_); + s = ReadAsync(opts, reader, read_len1, start_offset1, curr_); if (!s.ok()) { DestroyAndClearIOHandle(curr_); - bufs_[curr_].buffer_.Clear(); + bufs_[curr_].ClearBuffer(); return s; } explicit_prefetch_submitted_ = true; prev_len_ = 0; } + if (read_len2) { TEST_SYNC_POINT("FilePrefetchBuffer::PrefetchAsync:ExtraPrefetching"); - s = ReadAsync(opts, reader, read_len2, rounddown_start2, second); + s = ReadAsync(opts, reader, read_len2, start_offset2, second); if (!s.ok()) { DestroyAndClearIOHandle(second); - bufs_[second].buffer_.Clear(); + bufs_[second].ClearBuffer(); return s; } readahead_size_ = std::min(max_readahead_size_, readahead_size_ * 2); diff --git a/file/file_prefetch_buffer.h b/file/file_prefetch_buffer.h index ae8472496..0c6ba7e66 100644 --- a/file/file_prefetch_buffer.h +++ b/file/file_prefetch_buffer.h @@ -32,6 +32,11 @@ struct IOOptions; class RandomAccessFileReader; struct BufferInfo { + void ClearBuffer() { + buffer_.Clear(); + initial_end_offset_ = 0; + } + AlignedBuffer buffer_; uint64_t offset_ = 0; @@ -52,10 +57,23 @@ struct BufferInfo { // pos represents the index of this buffer in vector of BufferInfo. uint32_t pos_ = 0; + + // initial_end_offset is used to keep track of the end offset of the buffer + // that was originally called. It's helpful in case of autotuning of readahead + // size when callback is made to BlockBasedTableIterator. + // initial end offset of this buffer which will be the starting + // offset of next prefetch. + // + // For example - if end offset of previous buffer was 100 and because of + // readahead_size optimization, end_offset was trimmed to 60. Then for next + // prefetch call, start_offset should be intialized to 100 i.e start_offset = + // buf->initial_end_offset_. + uint64_t initial_end_offset_ = 0; }; enum class FilePrefetchBufferUsage { kTableOpenPrefetchTail, + kUserScanPrefetch, kUnknown, }; @@ -89,6 +107,7 @@ class FilePrefetchBuffer { bool implicit_auto_readahead = false, uint64_t num_file_reads = 0, uint64_t num_file_reads_for_auto_readahead = 0, FileSystem* fs = nullptr, SystemClock* clock = nullptr, Statistics* stats = nullptr, + const std::function& cb = nullptr, FilePrefetchBufferUsage usage = FilePrefetchBufferUsage::kUnknown) : curr_(0), readahead_size_(readahead_size), @@ -106,7 +125,8 @@ class FilePrefetchBuffer { fs_(fs), clock_(clock), stats_(stats), - usage_(usage) { + usage_(usage), + readaheadsize_cb_(cb) { assert((num_file_reads_ >= num_file_reads_for_auto_readahead_ + 1) || (num_file_reads_ == 0)); // If ReadOptions.async_io is enabled, data is asynchronously filled in @@ -183,14 +203,12 @@ class FilePrefetchBuffer { bool Enabled() const { return enable_; } // Load data into the buffer from a file. + // opts : the IO options to use. // reader : the file reader. // offset : the file offset to start reading from. // n : the number of bytes to read. - // rate_limiter_priority : rate limiting priority, or `Env::IO_TOTAL` to - // bypass. Status Prefetch(const IOOptions& opts, RandomAccessFileReader* reader, - uint64_t offset, size_t n, - Env::IOPriority rate_limiter_priority); + uint64_t offset, size_t n); // Request for reading the data from a file asynchronously. // If data already exists in the buffer, result will be updated. @@ -217,18 +235,14 @@ class FilePrefetchBuffer { // n : the number of bytes. // result : output buffer to put the data into. // s : output status. - // rate_limiter_priority : rate limiting priority, or `Env::IO_TOTAL` to - // bypass. // for_compaction : true if cache read is done for compaction read. bool TryReadFromCache(const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, size_t n, Slice* result, Status* s, - Env::IOPriority rate_limiter_priority, bool for_compaction = false); bool TryReadFromCacheAsync(const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, - size_t n, Slice* result, Status* status, - Env::IOPriority rate_limiter_priority); + size_t n, Slice* result, Status* status); // The minimum `offset` ever passed to TryReadFromCache(). This will nly be // tracked if track_min_offset = true. @@ -240,9 +254,6 @@ class FilePrefetchBuffer { void UpdateReadPattern(const uint64_t& offset, const size_t& len, bool decrease_readaheadsize) { if (decrease_readaheadsize) { - // Since this block was eligible for prefetch but it was found in - // cache, so check and decrease the readahead_size by 8KB (default) - // if eligible. DecreaseReadAheadIfEligible(offset, len); } prev_offset_ = offset; @@ -283,6 +294,12 @@ class FilePrefetchBuffer { // Callback function passed to underlying FS in case of asynchronous reads. void PrefetchAsyncCallback(const FSReadRequest& req, void* cb_arg); + void TEST_GetBufferOffsetandSize(uint32_t index, uint64_t& offset, + size_t& len) { + offset = bufs_[index].offset_; + len = bufs_[index].buffer_.CurrentSize(); + } + private: // Calculates roundoff offset and length to be prefetched based on alignment // and data present in buffer_. It also allocates new buffer or refit tail if @@ -295,25 +312,24 @@ class FilePrefetchBuffer { void AbortAllIOs(); - void UpdateBuffersIfNeeded(uint64_t offset); + void UpdateBuffersIfNeeded(uint64_t offset, size_t len); // It calls Poll API if any there is any pending asynchronous request. It then // checks if data is in any buffer. It clears the outdated data and swaps the // buffers if required. - void PollAndUpdateBuffersIfNeeded(uint64_t offset); + void PollAndUpdateBuffersIfNeeded(uint64_t offset, size_t len); Status PrefetchAsyncInternal(const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, size_t length, size_t readahead_size, - Env::IOPriority rate_limiter_priority, bool& copy_to_third_buffer); Status Read(const IOOptions& opts, RandomAccessFileReader* reader, - Env::IOPriority rate_limiter_priority, uint64_t read_len, - uint64_t chunk_len, uint64_t rounddown_start, uint32_t index); + uint64_t read_len, uint64_t chunk_len, uint64_t start_offset, + uint32_t index); Status ReadAsync(const IOOptions& opts, RandomAccessFileReader* reader, - uint64_t read_len, uint64_t rounddown_start, uint32_t index); + uint64_t read_len, uint64_t start_offset, uint32_t index); // Copy the data from src to third buffer. void CopyDataToBuffer(uint32_t src, uint64_t& offset, size_t& length); @@ -393,7 +409,13 @@ class FilePrefetchBuffer { bufs_[second].offset_)) { return false; } - bufs_[second].buffer_.Clear(); + + // Readahead size can be 0 because of trimming. + if (readahead_size_ == 0) { + return false; + } + + bufs_[second].ClearBuffer(); return true; } @@ -409,7 +431,6 @@ class FilePrefetchBuffer { Status HandleOverlappingData(const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, size_t length, size_t readahead_size, - Env::IOPriority rate_limiter_priority, bool& copy_to_third_buffer, uint64_t& tmp_offset, size_t& tmp_length); @@ -417,14 +438,35 @@ class FilePrefetchBuffer { RandomAccessFileReader* reader, uint64_t offset, size_t n, Slice* result, Status* s, - Env::IOPriority rate_limiter_priority, bool for_compaction = false); bool TryReadFromCacheAsyncUntracked(const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, size_t n, Slice* result, - Status* status, - Env::IOPriority rate_limiter_priority); + Status* status); + + void ReadAheadSizeTuning(bool read_curr_block, bool refit_tail, + uint64_t prev_buf_end_offset, uint32_t index, + size_t alignment, size_t length, + size_t readahead_size, uint64_t& offset, + uint64_t& end_offset, size_t& read_len, + uint64_t& chunk_len); + + void UpdateStats(bool found_in_buffer, size_t length_found) { + if (found_in_buffer) { + RecordTick(stats_, PREFETCH_HITS); + } + if (length_found > 0) { + RecordTick(stats_, PREFETCH_BYTES_USEFUL, length_found); + } + } + + void UpdateReadAheadTrimmedStat(size_t initial_length, + size_t updated_length) { + if (initial_length != updated_length) { + RecordTick(stats_, READAHEAD_TRIMMED); + } + } std::vector bufs_; // curr_ represents the index for bufs_ indicating which buffer is being @@ -467,5 +509,7 @@ class FilePrefetchBuffer { Statistics* stats_; FilePrefetchBufferUsage usage_; + + std::function readaheadsize_cb_; }; } // namespace ROCKSDB_NAMESPACE diff --git a/file/file_util.cc b/file/file_util.cc index 46faac67c..9eee10637 100644 --- a/file/file_util.cc +++ b/file/file_util.cc @@ -13,6 +13,7 @@ #include "file/sst_file_manager_impl.h" #include "file/writable_file_writer.h" #include "rocksdb/env.h" +#include "rocksdb/statistics.h" namespace ROCKSDB_NAMESPACE { @@ -137,7 +138,7 @@ IOStatus GenerateOneFileChecksum( std::string* file_checksum_func_name, size_t verify_checksums_readahead_size, bool /*allow_mmap_reads*/, std::shared_ptr& io_tracer, RateLimiter* rate_limiter, - Env::IOPriority rate_limiter_priority) { + const ReadOptions& read_options, Statistics* stats, SystemClock* clock) { if (checksum_factory == nullptr) { return IOStatus::InvalidArgument("Checksum factory is invalid"); } @@ -186,8 +187,8 @@ IOStatus GenerateOneFileChecksum( return io_s; } reader.reset(new RandomAccessFileReader( - std::move(r_file), file_path, nullptr /*Env*/, io_tracer, nullptr, - Histograms::HISTOGRAM_ENUM_MAX, nullptr, rate_limiter)); + std::move(r_file), file_path, clock, io_tracer, stats, + Histograms::SST_READ_MICROS, nullptr, rate_limiter)); } // Found that 256 KB readahead size provides the best performance, based on @@ -206,11 +207,15 @@ IOStatus GenerateOneFileChecksum( Slice slice; uint64_t offset = 0; IOOptions opts; + io_s = reader->PrepareIOOptions(read_options, opts); + if (!io_s.ok()) { + return io_s; + } while (size > 0) { size_t bytes_to_read = static_cast(std::min(uint64_t{readahead_size}, size)); - io_s = reader->Read(opts, offset, bytes_to_read, &slice, buf.get(), nullptr, - rate_limiter_priority); + io_s = + reader->Read(opts, offset, bytes_to_read, &slice, buf.get(), nullptr); if (!io_s.ok()) { return IOStatus::Corruption("file read failed with error: " + io_s.ToString()); diff --git a/file/file_util.h b/file/file_util.h index 8b59731eb..9c95478c7 100644 --- a/file/file_util.h +++ b/file/file_util.h @@ -11,6 +11,7 @@ #include "rocksdb/env.h" #include "rocksdb/file_system.h" #include "rocksdb/sst_file_writer.h" +#include "rocksdb/statistics.h" #include "rocksdb/status.h" #include "rocksdb/system_clock.h" #include "rocksdb/types.h" @@ -52,6 +53,7 @@ extern Status DeleteDBFile(const ImmutableDBOptions* db_options, const std::string& path_to_sync, const bool force_bg, const bool force_fg); +// TODO(hx235): pass the whole DBOptions intead of its individual fields extern IOStatus GenerateOneFileChecksum( FileSystem* fs, const std::string& file_path, FileChecksumGenFactory* checksum_factory, @@ -59,7 +61,7 @@ extern IOStatus GenerateOneFileChecksum( std::string* file_checksum_func_name, size_t verify_checksums_readahead_size, bool allow_mmap_reads, std::shared_ptr& io_tracer, RateLimiter* rate_limiter, - Env::IOPriority rate_limiter_priority); + const ReadOptions& read_options, Statistics* stats, SystemClock* clock); inline IOStatus PrepareIOFromReadOptions(const ReadOptions& ro, SystemClock* clock, IOOptions& opts) { diff --git a/file/filename.cc b/file/filename.cc index 1e04c7339..fb7d25472 100644 --- a/file/filename.cc +++ b/file/filename.cc @@ -8,10 +8,9 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "file/filename.h" -#include -#include - +#include #include +#include #include #include "file/writable_file_writer.h" diff --git a/file/prefetch_test.cc b/file/prefetch_test.cc index 6e807f581..b97564129 100644 --- a/file/prefetch_test.cc +++ b/file/prefetch_test.cc @@ -169,18 +169,18 @@ TEST_P(PrefetchTest, Basic) { // create first key range WriteBatch batch; for (int i = 0; i < kNumKeys; i++) { - ASSERT_OK(batch.Put(BuildKey(i), "value for range 1 key")); + ASSERT_OK(batch.Put(BuildKey(i), "v1")); } ASSERT_OK(db_->Write(WriteOptions(), &batch)); - ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(db_->Flush(FlushOptions())); // create second key range batch.Clear(); for (int i = 0; i < kNumKeys; i++) { - ASSERT_OK(batch.Put(BuildKey(i, "key2"), "value for range 2 key")); + ASSERT_OK(batch.Put(BuildKey(i, "key2"), "v2")); } ASSERT_OK(db_->Write(WriteOptions(), &batch)); - ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(db_->Flush(FlushOptions())); // delete second key range batch.Clear(); @@ -190,16 +190,19 @@ TEST_P(PrefetchTest, Basic) { ASSERT_OK(db_->Write(WriteOptions(), &batch)); ASSERT_OK(db_->Flush(FlushOptions())); + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + const size_t num_file = metadata.size(); // To verify SST file tail prefetch (once per file) during flush output // verification if (support_prefetch && !use_direct_io) { ASSERT_TRUE(fs->IsPrefetchCalled()); - ASSERT_EQ(3, fs->GetPrefetchCount()); + ASSERT_EQ(num_file, fs->GetPrefetchCount()); ASSERT_EQ(0, buff_prefetch_count); fs->ClearPrefetchCount(); } else { ASSERT_FALSE(fs->IsPrefetchCalled()); - ASSERT_EQ(buff_prefetch_count, 3); + ASSERT_EQ(buff_prefetch_count, num_file); buff_prefetch_count = 0; } @@ -238,16 +241,9 @@ TEST_P(PrefetchTest, Basic) { fs->ClearPrefetchCount(); } else { ASSERT_FALSE(fs->IsPrefetchCalled()); - if (use_direct_io) { - // To rule out false positive by the SST file tail prefetch during - // compaction output verification - ASSERT_GT(buff_prefetch_count, 1); - } else { - // In buffered IO, compaction readahead size is 0, leading to no prefetch - // during compaction input read - ASSERT_EQ(buff_prefetch_count, 1); - } - + // To rule out false positive by the SST file tail prefetch during + // compaction output verification + ASSERT_GT(buff_prefetch_count, 1); buff_prefetch_count = 0; ASSERT_GT(cur_table_open_prefetch_tail_read.count, @@ -265,6 +261,7 @@ TEST_P(PrefetchTest, Basic) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { num_keys++; } + ASSERT_OK(iter->status()); (void)num_keys; } @@ -425,7 +422,8 @@ TEST_P(PrefetchTailTest, UpgradeToTailSizeInManifest) { } if (UseDirectIO()) { ROCKSDB_GTEST_BYPASS( - "To simplify testing logics with setting file's buffer alignment to be " + "To simplify testing logics with setting file's buffer alignment to " + "be " "1, direct IO is required to be disabled."); } @@ -462,8 +460,8 @@ TEST_P(PrefetchTailTest, UpgradeToTailSizeInManifest) { // inferred to be a small number for files with no tail size recorded in // manifest. // "1" is chosen to be such number so that with `small_buffer_alignment == - // true` and `use_small_cache == true`, it would have caused one file read per - // index partition during db open if the upgrade is done wrong. + // true` and `use_small_cache == true`, it would have caused one file read + // per index partition during db open if the upgrade is done wrong. SyncPoint::GetInstance()->SetCallBack( "BlockBasedTable::Open::TailPrefetchLen", [&](void* arg) { std::pair* prefetch_off_len_pair = @@ -488,8 +486,8 @@ TEST_P(PrefetchTailTest, UpgradeToTailSizeInManifest) { int64_t num_index_partition = GetNumIndexPartition(); // If the upgrade is done right, db open will prefetch all the index // partitions at once, instead of doing one read per partition. - // That is, together with `metadata_block_size == 1`, there will be more index - // partitions than number of non index partitions reads. + // That is, together with `metadata_block_size == 1`, there will be more + // index partitions than number of non index partitions reads. ASSERT_LT(db_open_file_read.count, num_index_partition); Close(); @@ -557,7 +555,7 @@ TEST_P(PrefetchTest, ConfigureAutoMaxReadaheadSize) { } Close(); std::vector buff_prefectch_level_count = {0, 0, 0}; - TryReopen(options); + ASSERT_OK(TryReopen(options)); { auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); fs->ClearPrefetchCount(); @@ -657,9 +655,6 @@ TEST_P(PrefetchTest, ConfigureInternalAutoReadaheadSize) { SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", [&](void*) { buff_prefetch_count++; }); - - SyncPoint::GetInstance()->EnableProcessing(); - SyncPoint::GetInstance()->EnableProcessing(); Status s = TryReopen(options); @@ -685,7 +680,7 @@ TEST_P(PrefetchTest, ConfigureInternalAutoReadaheadSize) { } Close(); - TryReopen(options); + ASSERT_OK(TryReopen(options)); { auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); fs->ClearPrefetchCount(); @@ -702,8 +697,8 @@ TEST_P(PrefetchTest, ConfigureInternalAutoReadaheadSize) { "{initial_auto_readahead_size=0;}"}})); break; case 1: - // intial_auto_readahead_size and max_auto_readahead_size are set same - // so readahead_size remains same. + // intial_auto_readahead_size and max_auto_readahead_size are set + // same so readahead_size remains same. ASSERT_OK(db_->SetOptions({{"block_based_table_factory", "{initial_auto_readahead_size=4096;max_" "auto_readahead_size=4096;}"}})); @@ -721,6 +716,7 @@ TEST_P(PrefetchTest, ConfigureInternalAutoReadaheadSize) { iter->Seek(Key(key_count++)); iter->Next(); } + ASSERT_OK(iter->status()); buff_prefetch_level_count[level] = buff_prefetch_count; if (support_prefetch && !use_direct_io) { @@ -800,7 +796,7 @@ TEST_P(PrefetchTest, ConfigureNumFilesReadsForReadaheadSize) { ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); Close(); - TryReopen(options); + ASSERT_OK(TryReopen(options)); fs->ClearPrefetchCount(); buff_prefetch_count = 0; @@ -810,8 +806,9 @@ TEST_P(PrefetchTest, ConfigureNumFilesReadsForReadaheadSize) { /* * Reseek keys from sequential Data Blocks within same partitioned * index. It will prefetch the data block at the first seek since - * num_file_reads_for_auto_readahead = 0. Data Block size is nearly 4076 so - * readahead will fetch 8 * 1024 data more initially (2 more data blocks). + * num_file_reads_for_auto_readahead = 0. Data Block size is nearly 4076 + * so readahead will fetch 8 * 1024 data more initially (2 more data + * blocks). */ iter->Seek(BuildKey(0)); // Prefetch data + index block since // num_file_reads_for_auto_readahead = 0. @@ -909,8 +906,8 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) { /* * Reseek keys from sequential Data Blocks within same partitioned * index. After 2 sequential reads it will prefetch the data block. - * Data Block size is nearly 4076 so readahead will fetch 8 * 1024 data more - * initially (2 more data blocks). + * Data Block size is nearly 4076 so readahead will fetch 8 * 1024 data + * more initially (2 more data blocks). */ iter->Seek(BuildKey(0)); ASSERT_TRUE(iter->Valid()); @@ -987,9 +984,9 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) { { /* * Reseek keys from sequential data blocks to set implicit auto readahead - * and prefetch data but after that iterate over different (non sequential) - * data blocks which won't prefetch any data further. So buff_prefetch_count - * will be 1 for the first one. + * and prefetch data but after that iterate over different (non + * sequential) data blocks which won't prefetch any data further. So + * buff_prefetch_count will be 1 for the first one. */ auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); iter->Seek(BuildKey(0)); @@ -1016,8 +1013,8 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) { buff_prefetch_count = 0; } - // Read sequentially to confirm readahead_size is reset to initial value (2 - // more data blocks) + // Read sequentially to confirm readahead_size is reset to initial value + // (2 more data blocks) iter->Seek(BuildKey(1011)); ASSERT_TRUE(iter->Valid()); iter->Seek(BuildKey(1015)); @@ -1067,8 +1064,8 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) { } { /* - * Reseek over different keys from different blocks. buff_prefetch_count is - * set 0. + * Reseek over different keys from different blocks. buff_prefetch_count + * is set 0. */ auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); int i = 0; @@ -1076,6 +1073,7 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) { do { iter->Seek(BuildKey(i)); if (!iter->Valid()) { + ASSERT_OK(iter->status()); break; } i = i + 100; @@ -1095,6 +1093,7 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) { auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { } + ASSERT_OK(iter->status()); if (support_prefetch && !use_direct_io) { ASSERT_EQ(fs->GetPrefetchCount(), 13); fs->ClearPrefetchCount(); @@ -1172,8 +1171,8 @@ TEST_P(PrefetchTest, PrefetchWhenReseekwithCache) { /* * Reseek keys from sequential Data Blocks within same partitioned * index. After 2 sequential reads it will prefetch the data block. - * Data Block size is nearly 4076 so readahead will fetch 8 * 1024 data more - * initially (2 more data blocks). + * Data Block size is nearly 4076 so readahead will fetch 8 * 1024 data + * more initially (2 more data blocks). */ auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); // Warm up the cache @@ -1200,8 +1199,8 @@ TEST_P(PrefetchTest, PrefetchWhenReseekwithCache) { ASSERT_TRUE(iter->Valid()); iter->Seek(BuildKey(1004)); // Prefetch data (not in cache). ASSERT_TRUE(iter->Valid()); - // Missed one sequential block but next is in already in buffer so readahead - // will not be reset. + // Missed one sequential block but next is in already in buffer so + // readahead will not be reset. iter->Seek(BuildKey(1011)); ASSERT_TRUE(iter->Valid()); // Prefetch data but blocks are in cache so no prefetch and reset. @@ -1235,6 +1234,267 @@ TEST_P(PrefetchTest, PrefetchWhenReseekwithCache) { Close(); } +TEST_P(PrefetchTest, PrefetchWithBlockLookupAutoTuneTest) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + + std::shared_ptr fs = + std::make_shared(FileSystem::Default(), false); + + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + Options options; + SetGenericOptions(env.get(), /*use_direct_io=*/false, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + ASSERT_OK(s); + + Random rnd(309); + WriteBatch batch; + + for (int i = 0; i < 26; i++) { + std::string key = "my_key_"; + + for (int j = 0; j < 10; j++) { + key += char('a' + i); + ASSERT_OK(batch.Put(key, rnd.RandomString(1000))); + } + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = "my_key_a"; + + std::string end_key = "my_key_"; + for (int j = 0; j < 10; j++) { + end_key += char('a' + 25); + } + + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + // Try with different num_file_reads_for_auto_readahead from 0 to 3. + for (size_t i = 0; i < 3; i++) { + std::shared_ptr cache = NewLRUCache(1024 * 1024, 2); + table_options.block_cache = cache; + table_options.no_block_cache = false; + table_options.num_file_reads_for_auto_readahead = i; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + s = TryReopen(options); + ASSERT_OK(s); + + // Warm up the cache. + { + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + + iter->Seek("my_key_bbb"); + ASSERT_TRUE(iter->Valid()); + + iter->Seek("my_key_ccccccccc"); + ASSERT_TRUE(iter->Valid()); + + iter->Seek("my_key_ddd"); + ASSERT_TRUE(iter->Valid()); + + iter->Seek("my_key_ddddddd"); + ASSERT_TRUE(iter->Valid()); + + iter->Seek("my_key_e"); + ASSERT_TRUE(iter->Valid()); + + iter->Seek("my_key_eeeee"); + ASSERT_TRUE(iter->Valid()); + + iter->Seek("my_key_eeeeeeeee"); + ASSERT_TRUE(iter->Valid()); + } + + ReadOptions ropts; + ropts.auto_readahead_size = true; + ReadOptions cmp_ro; + cmp_ro.auto_readahead_size = false; + + if (std::get<0>(GetParam())) { + ropts.readahead_size = cmp_ro.readahead_size = 32768; + } + + if (std::get<1>(GetParam())) { + ropts.async_io = true; + } + + // With and without tuning readahead_size. + { + ASSERT_OK(options.statistics->Reset()); + // Seek. + { + Slice ub = Slice("my_key_uuu"); + Slice* ub_ptr = &ub; + cmp_ro.iterate_upper_bound = ub_ptr; + ropts.iterate_upper_bound = ub_ptr; + + auto iter = std::unique_ptr(db_->NewIterator(ropts)); + auto cmp_iter = std::unique_ptr(db_->NewIterator(cmp_ro)); + + Slice seek_key = Slice("my_key_aaa"); + iter->Seek(seek_key); + cmp_iter->Seek(seek_key); + + while (iter->Valid() && cmp_iter->Valid()) { + if (iter->key() != cmp_iter->key()) { + // Error + ASSERT_TRUE(false); + } + iter->Next(); + cmp_iter->Next(); + } + + ASSERT_OK(cmp_iter->status()); + ASSERT_OK(iter->status()); + } + + // Reseek with new upper_bound_iterator. + { + Slice ub = Slice("my_key_y"); + ropts.iterate_upper_bound = &ub; + cmp_ro.iterate_upper_bound = &ub; + + auto iter = std::unique_ptr(db_->NewIterator(ropts)); + auto cmp_iter = std::unique_ptr(db_->NewIterator(cmp_ro)); + + Slice reseek_key = Slice("my_key_v"); + iter->Seek(reseek_key); + cmp_iter->Seek(reseek_key); + + while (iter->Valid() && cmp_iter->Valid()) { + if (iter->key() != cmp_iter->key()) { + // Error + ASSERT_TRUE(false); + } + iter->Next(); + cmp_iter->Next(); + } + + ASSERT_OK(cmp_iter->status()); + ASSERT_OK(iter->status()); + } + } + Close(); + } +} + +TEST_F(PrefetchTest, PrefetchWithBlockLookupAutoTuneWithPrev) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + + // First param is if the mockFS support_prefetch or not + std::shared_ptr fs = + std::make_shared(FileSystem::Default(), false); + + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + Options options; + SetGenericOptions(env.get(), /*use_direct_io=*/false, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + std::shared_ptr cache = NewLRUCache(1024 * 1024, 2); + table_options.block_cache = cache; + table_options.no_block_cache = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + ASSERT_OK(s); + + Random rnd(309); + WriteBatch batch; + + for (int i = 0; i < 26; i++) { + std::string key = "my_key_"; + + for (int j = 0; j < 10; j++) { + key += char('a' + i); + ASSERT_OK(batch.Put(key, rnd.RandomString(1000))); + } + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = "my_key_a"; + + std::string end_key = "my_key_"; + for (int j = 0; j < 10; j++) { + end_key += char('a' + 25); + } + + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + ReadOptions ropts; + ropts.auto_readahead_size = true; + + { + // Seek. + Slice ub = Slice("my_key_uuu"); + Slice* ub_ptr = &ub; + ropts.iterate_upper_bound = ub_ptr; + ropts.auto_readahead_size = true; + + ReadOptions cmp_readopts = ropts; + cmp_readopts.auto_readahead_size = false; + + auto iter = std::unique_ptr(db_->NewIterator(ropts)); + auto cmp_iter = std::unique_ptr(db_->NewIterator(cmp_readopts)); + + Slice seek_key = Slice("my_key_bbb"); + { + cmp_iter->Seek(seek_key); + ASSERT_TRUE(cmp_iter->Valid()); + ASSERT_OK(cmp_iter->status()); + + iter->Seek(seek_key); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + + ASSERT_EQ(iter->key(), cmp_iter->key()); + } + + // Prev op should pass with auto tuning of readahead_size. + { + cmp_iter->Prev(); + ASSERT_TRUE(cmp_iter->Valid()); + ASSERT_OK(cmp_iter->status()); + + iter->Prev(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + + ASSERT_EQ(iter->key(), cmp_iter->key()); + } + + // Reseek would follow as usual. + { + cmp_iter->Seek(seek_key); + ASSERT_TRUE(cmp_iter->Valid()); + ASSERT_OK(cmp_iter->status()); + + iter->Seek(seek_key); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), cmp_iter->key()); + } + } + Close(); +} + // This test verifies the functionality of ReadOptions.adaptive_readahead. TEST_P(PrefetchTest, DBIterLevelReadAhead) { const int kNumKeys = 1000; @@ -1317,6 +1577,7 @@ TEST_P(PrefetchTest, DBIterLevelReadAhead) { ASSERT_OK(iter->status()); num_keys++; } + ASSERT_OK(iter->status()); ASSERT_EQ(num_keys, total_keys); // For index and data blocks. @@ -1431,6 +1692,7 @@ TEST_P(PrefetchTest, DBIterLevelReadAheadWithAsyncIO) { ASSERT_OK(iter->status()); num_keys++; } + ASSERT_OK(iter->status()); ASSERT_EQ(num_keys, total_keys); // For index and data blocks. @@ -1461,6 +1723,69 @@ TEST_P(PrefetchTest, DBIterLevelReadAheadWithAsyncIO) { Close(); } +TEST_P(PrefetchTest, AvoidBlockCacheLookupTwice) { + const int kNumKeys = 1000; + // Set options + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + bool use_direct_io = std::get<0>(GetParam()); + bool async_io = std::get<1>(GetParam()); + + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); // 8MB + table_options.block_cache = cache; + table_options.no_block_cache = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + // Write to DB. + { + WriteBatch batch; + Random rnd(309); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + } + + ReadOptions ro; + ro.async_io = async_io; + // Iterate over the keys. + { + // Each block contains around 4 keys. + auto iter = std::unique_ptr(db_->NewIterator(ro)); + ASSERT_OK(options.statistics->Reset()); + + iter->Seek(BuildKey(99)); // Prefetch data because of seek parallelization. + ASSERT_TRUE(iter->Valid()); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS), + 1); + } + + Close(); +} + TEST_P(PrefetchTest, DBIterAsyncIONoIOUring) { if (mem_env_ || encrypted_env_) { ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); @@ -1518,6 +1843,7 @@ TEST_P(PrefetchTest, DBIterAsyncIONoIOUring) { ASSERT_OK(iter->status()); num_keys++; } + ASSERT_OK(iter->status()); ASSERT_EQ(num_keys, total_keys); // Check stats to make sure async prefetch is done. @@ -1545,6 +1871,7 @@ TEST_P(PrefetchTest, DBIterAsyncIONoIOUring) { ASSERT_OK(iter->status()); num_keys++; } + ASSERT_OK(iter->status()); ASSERT_EQ(num_keys, total_keys); // Check stats to make sure async prefetch is done. @@ -1655,12 +1982,13 @@ TEST_P(PrefetchTest1, SeekWithExtraPrefetchAsyncIO) { 0)); // Prefetch data on seek because of seek parallelization. ASSERT_TRUE(iter->Valid()); - // Do extra prefetching in Seek only if num_file_reads_for_auto_readahead - // = 0. + // Do extra prefetching in Seek only if + // num_file_reads_for_auto_readahead = 0. ASSERT_EQ(extra_prefetch_buff_cnt, (i == 0 ? 1 : 0)); // buff_prefetch_count is 2 because of index block when // num_file_reads_for_auto_readahead = 0. - // If num_file_reads_for_auto_readahead > 0, index block isn't prefetched. + // If num_file_reads_for_auto_readahead > 0, index block isn't + // prefetched. ASSERT_EQ(buff_prefetch_count, i == 0 ? 2 : 1); extra_prefetch_buff_cnt = 0; @@ -1669,8 +1997,8 @@ TEST_P(PrefetchTest1, SeekWithExtraPrefetchAsyncIO) { iter->Seek( BuildKey(22)); // Prefetch data because of seek parallelization. ASSERT_TRUE(iter->Valid()); - // Do extra prefetching in Seek only if num_file_reads_for_auto_readahead - // = 0. + // Do extra prefetching in Seek only if + // num_file_reads_for_auto_readahead = 0. ASSERT_EQ(extra_prefetch_buff_cnt, (i == 0 ? 1 : 0)); ASSERT_EQ(buff_prefetch_count, 1); @@ -1680,8 +2008,8 @@ TEST_P(PrefetchTest1, SeekWithExtraPrefetchAsyncIO) { iter->Seek( BuildKey(33)); // Prefetch data because of seek parallelization. ASSERT_TRUE(iter->Valid()); - // Do extra prefetching in Seek only if num_file_reads_for_auto_readahead - // = 0. + // Do extra prefetching in Seek only if + // num_file_reads_for_auto_readahead = 0. ASSERT_EQ(extra_prefetch_buff_cnt, (i == 0 ? 1 : 0)); ASSERT_EQ(buff_prefetch_count, 1); } @@ -1772,8 +2100,8 @@ TEST_P(PrefetchTest1, NonSequentialReadsWithAdaptiveReadahead) { Close(); } -// This test verifies the functionality of adaptive_readaheadsize with cache and -// if block is found in cache, decrease the readahead_size if +// This test verifies the functionality of adaptive_readaheadsize with cache +// and if block is found in cache, decrease the readahead_size if // - its enabled internally by RocksDB (implicit_auto_readahead_) and, // - readahead_size is greater than 0 and, // - the block would have called prefetch API if not found in cache for @@ -1895,8 +2223,8 @@ TEST_P(PrefetchTest1, DecreaseReadAheadIfInCache) { ASSERT_TRUE(iter->Valid()); // Prefetch data (not in buffer) but found in cache. So decrease - // readahead_size. Since it will 0 after decrementing so readahead_size will - // be set to initial value. + // readahead_size. Since it will 0 after decrementing so readahead_size + // will be set to initial value. iter->Seek(BuildKey(1019)); ASSERT_TRUE(iter->Valid()); expected_current_readahead_size = std::max( @@ -2008,8 +2336,8 @@ TEST_P(PrefetchTest1, SeekParallelizationTest) { HistogramData async_read_bytes; options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); - // not all platforms support io_uring. In that case it'll fallback to normal - // prefetching without async_io. + // not all platforms support io_uring. In that case it'll fallback to + // normal prefetching without async_io. if (read_async_called) { ASSERT_EQ(buff_prefetch_async_count, 2); ASSERT_GT(async_read_bytes.count, 0); @@ -2123,6 +2451,7 @@ TEST_P(PrefetchTest, ReadAsyncWithPosixFS) { ASSERT_OK(iter->status()); num_keys++; } + ASSERT_OK(iter->status()); if (read_async_called) { ASSERT_EQ(num_keys, total_keys); @@ -2507,6 +2836,7 @@ TEST_P(PrefetchTest, TraceReadAsyncWithCallbackWrapper) { ASSERT_OK(iter->status()); num_keys++; } + ASSERT_OK(iter->status()); // End the tracing. ASSERT_OK(db_->EndIOTrace()); @@ -2613,8 +2943,91 @@ TEST_F(FilePrefetchBufferTest, SeekWithBlockCacheHit) { fpb.UpdateReadPattern(0, 4096, false); // Now read some data that straddles the two prefetch buffers - offset 8192 to // 16384 - ASSERT_TRUE(fpb.TryReadFromCacheAsync(IOOptions(), r.get(), 8192, 8192, - &result, &s, Env::IOPriority::IO_LOW)); + IOOptions io_opts; + io_opts.rate_limiter_priority = Env::IOPriority::IO_LOW; + ASSERT_TRUE( + fpb.TryReadFromCacheAsync(io_opts, r.get(), 8192, 8192, &result, &s)); +} + +// Test to ensure when PrefetchAsync is called during seek, it doesn't do any +// alignment or prefetch extra if readahead is not enabled during seek. +TEST_F(FilePrefetchBufferTest, SeekWithoutAlignment) { + std::string fname = "seek-wwithout-alignment"; + Random rand(0); + std::string content = rand.RandomString(32768); + Write(fname, content); + + FileOptions opts; + std::unique_ptr r; + Read(fname, opts, &r); + + size_t alignment = r->file()->GetRequiredBufferAlignment(); + size_t n = alignment / 2; + + int read_async_called = 0; + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::ReadAsync", + [&](void* /*arg*/) { read_async_called++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Without readahead enabled, there will be no alignment and offset of buffer + // will be n. + { + FilePrefetchBuffer fpb( + /*readahead_size=*/8192, /*max_readahead_size=*/16384, /*enable=*/true, + /*track_min_offset=*/false, /*implicit_auto_readahead=*/true, + /*num_file_reads=*/0, /*num_file_reads_for_auto_readahead=*/2, fs()); + + Slice result; + // Simulate a seek of half of alignment bytes at offset n. Due to the + // readahead settings, it won't prefetch extra or do any alignment and + // offset of buffer will be n. + Status s = fpb.PrefetchAsync(IOOptions(), r.get(), n, n, &result); + + // Platforms that don't have IO uring may not support async IO. + if (s.IsNotSupported()) { + return; + } + + ASSERT_TRUE(s.IsTryAgain()); + + IOOptions io_opts; + io_opts.rate_limiter_priority = Env::IOPriority::IO_LOW; + ASSERT_TRUE(fpb.TryReadFromCacheAsync(io_opts, r.get(), n, n, &result, &s)); + + if (read_async_called) { + ASSERT_EQ(fpb.GetPrefetchOffset(), n); + } + } + + // With readahead enabled, it will do the alignment and prefetch and offset of + // buffer will be 0. + { + read_async_called = false; + FilePrefetchBuffer fpb( + /*readahead_size=*/16384, /*max_readahead_size=*/16384, /*enable=*/true, + /*track_min_offset=*/false, /*implicit_auto_readahead=*/false, + /*num_file_reads=*/0, /*num_file_reads_for_auto_readahead=*/2, fs()); + + Slice result; + // Simulate a seek of half of alignment bytes at offset n. + Status s = fpb.PrefetchAsync(IOOptions(), r.get(), n, n, &result); + + // Platforms that don't have IO uring may not support async IO. + if (s.IsNotSupported()) { + return; + } + + ASSERT_TRUE(s.IsTryAgain()); + + IOOptions io_opts; + io_opts.rate_limiter_priority = Env::IOPriority::IO_LOW; + ASSERT_TRUE(fpb.TryReadFromCacheAsync(io_opts, r.get(), n, n, &result, &s)); + + if (read_async_called) { + ASSERT_EQ(fpb.GetPrefetchOffset(), 0); + } + } } TEST_F(FilePrefetchBufferTest, NoSyncWithAsyncIO) { @@ -2649,9 +3062,10 @@ TEST_F(FilePrefetchBufferTest, NoSyncWithAsyncIO) { } ASSERT_TRUE(s.IsTryAgain()); - ASSERT_TRUE(fpb.TryReadFromCacheAsync(IOOptions(), r.get(), /*offset=*/3000, - /*length=*/4000, &async_result, &s, - Env::IOPriority::IO_LOW)); + IOOptions io_opts; + io_opts.rate_limiter_priority = Env::IOPriority::IO_LOW; + ASSERT_TRUE(fpb.TryReadFromCacheAsync(io_opts, r.get(), /*offset=*/3000, + /*length=*/4000, &async_result, &s)); // No sync call should be made. HistogramData sst_read_micros; stats()->histogramData(SST_READ_MICROS, &sst_read_micros); @@ -2662,11 +3076,50 @@ TEST_F(FilePrefetchBufferTest, NoSyncWithAsyncIO) { // Length should be 4000. ASSERT_EQ(async_result.size(), 4000); // Data correctness. - Slice result(content.c_str() + 3000, 4000); + Slice result(&content[3000], 4000); ASSERT_EQ(result.size(), 4000); ASSERT_EQ(result, async_result); } +TEST_F(FilePrefetchBufferTest, SyncReadaheadStats) { + std::string fname = "seek-with-block-cache-hit"; + Random rand(0); + std::string content = rand.RandomString(32768); + Write(fname, content); + + FileOptions opts; + std::unique_ptr r; + Read(fname, opts, &r); + + std::shared_ptr stats = CreateDBStatistics(); + FilePrefetchBuffer fpb(8192, 8192, true, false, false, 0, 0, fs(), nullptr, + stats.get()); + Slice result; + // Simulate a seek of 4096 bytes at offset 0. Due to the readahead settings, + // it will do two reads of 4096+8192 and 8192 + Status s; + ASSERT_TRUE(fpb.TryReadFromCache(IOOptions(), r.get(), 0, 4096, &result, &s)); + ASSERT_EQ(s, Status::OK()); + ASSERT_EQ(stats->getTickerCount(PREFETCH_HITS), 0); + ASSERT_EQ(stats->getTickerCount(PREFETCH_BYTES_USEFUL), 0); + + // Simulate a block cache hit + fpb.UpdateReadPattern(4096, 4096, false); + // Now read some data that straddles the two prefetch buffers - offset 8192 to + // 16384 + ASSERT_TRUE( + fpb.TryReadFromCache(IOOptions(), r.get(), 8192, 8192, &result, &s)); + ASSERT_EQ(s, Status::OK()); + ASSERT_EQ(stats->getTickerCount(PREFETCH_HITS), 0); + ASSERT_EQ(stats->getTickerCount(PREFETCH_BYTES_USEFUL), 4096); + + ASSERT_TRUE( + fpb.TryReadFromCache(IOOptions(), r.get(), 12288, 4096, &result, &s)); + ASSERT_EQ(s, Status::OK()); + ASSERT_EQ(stats->getTickerCount(PREFETCH_HITS), 1); + ASSERT_EQ(stats->getTickerCount(PREFETCH_BYTES_USEFUL), 8192); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/file/random_access_file_reader.cc b/file/random_access_file_reader.cc index e38e9ec13..5de9d1305 100644 --- a/file/random_access_file_reader.cc +++ b/file/random_access_file_reader.cc @@ -22,12 +22,37 @@ #include "util/rate_limiter_impl.h" namespace ROCKSDB_NAMESPACE { -const std::array - kReadHistograms{{ - FILE_READ_FLUSH_MICROS, - FILE_READ_COMPACTION_MICROS, - FILE_READ_DB_OPEN_MICROS, - }}; +inline Histograms GetFileReadHistograms(Statistics* stats, + Env::IOActivity io_activity) { + switch (io_activity) { + case Env::IOActivity::kFlush: + return Histograms::FILE_READ_FLUSH_MICROS; + case Env::IOActivity::kCompaction: + return Histograms::FILE_READ_COMPACTION_MICROS; + case Env::IOActivity::kDBOpen: + return Histograms::FILE_READ_DB_OPEN_MICROS; + default: + break; + } + + if (stats && stats->get_stats_level() > StatsLevel::kExceptDetailedTimers) { + switch (io_activity) { + case Env::IOActivity::kGet: + return Histograms::FILE_READ_GET_MICROS; + case Env::IOActivity::kMultiGet: + return Histograms::FILE_READ_MULTIGET_MICROS; + case Env::IOActivity::kDBIterator: + return Histograms::FILE_READ_DB_ITERATOR_MICROS; + case Env::IOActivity::kVerifyDBChecksum: + return Histograms::FILE_READ_VERIFY_DB_CHECKSUM_MICROS; + case Env::IOActivity::kVerifyFileChecksums: + return Histograms::FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS; + default: + break; + } + } + return Histograms::HISTOGRAM_ENUM_MAX; +} inline void RecordIOStats(Statistics* stats, Temperature file_temperature, bool is_last_level, size_t size) { IOSTATS_ADD(bytes_read, size); @@ -79,11 +104,11 @@ IOStatus RandomAccessFileReader::Create( return io_s; } -IOStatus RandomAccessFileReader::Read( - const IOOptions& opts, uint64_t offset, size_t n, Slice* result, - char* scratch, AlignedBuf* aligned_buf, - Env::IOPriority rate_limiter_priority) const { +IOStatus RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset, + size_t n, Slice* result, char* scratch, + AlignedBuf* aligned_buf) const { (void)aligned_buf; + const Env::IOPriority rate_limiter_priority = opts.rate_limiter_priority; TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read", nullptr); @@ -108,9 +133,7 @@ IOStatus RandomAccessFileReader::Read( { StopWatch sw(clock_, stats_, hist_type_, - (opts.io_activity != Env::IOActivity::kUnknown) - ? kReadHistograms[(std::size_t)(opts.io_activity)] - : Histograms::HISTOGRAM_ENUM_MAX, + GetFileReadHistograms(stats_, opts.io_activity), (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/, true /*delay_enabled*/); auto prev_perf_level = GetPerfLevel(); @@ -248,6 +271,14 @@ IOStatus RandomAccessFileReader::Read( file_read_hist_->Add(elapsed); } +#ifndef NDEBUG + auto pair = std::make_pair(&file_name_, &io_s); + if (offset == 0) { + TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read::BeforeReturn", + &pair); + } + TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read::AnyOffset", &pair); +#endif return io_s; } @@ -277,9 +308,10 @@ bool TryMerge(FSReadRequest* dest, const FSReadRequest& src) { return true; } -IOStatus RandomAccessFileReader::MultiRead( - const IOOptions& opts, FSReadRequest* read_reqs, size_t num_reqs, - AlignedBuf* aligned_buf, Env::IOPriority rate_limiter_priority) const { +IOStatus RandomAccessFileReader::MultiRead(const IOOptions& opts, + FSReadRequest* read_reqs, + size_t num_reqs, + AlignedBuf* aligned_buf) const { (void)aligned_buf; // suppress warning of unused variable in LITE mode assert(num_reqs > 0); @@ -288,6 +320,7 @@ IOStatus RandomAccessFileReader::MultiRead( assert(read_reqs[i].offset <= read_reqs[i + 1].offset); } #endif // !NDEBUG + const Env::IOPriority rate_limiter_priority = opts.rate_limiter_priority; // To be paranoid modify scratch a little bit, so in case underlying // FileSystem doesn't fill the buffer but return success and `scratch` returns @@ -304,9 +337,7 @@ IOStatus RandomAccessFileReader::MultiRead( uint64_t elapsed = 0; { StopWatch sw(clock_, stats_, hist_type_, - (opts.io_activity != Env::IOActivity::kUnknown) - ? kReadHistograms[(std::size_t)(opts.io_activity)] - : Histograms::HISTOGRAM_ENUM_MAX, + GetFileReadHistograms(stats_, opts.io_activity), (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/, true /*delay_enabled*/); auto prev_perf_level = GetPerfLevel(); @@ -461,8 +492,9 @@ IOStatus RandomAccessFileReader::ReadAsync( auto read_async_callback = std::bind(&RandomAccessFileReader::ReadAsyncCallback, this, std::placeholders::_1, std::placeholders::_2); - ReadAsyncInfo* read_async_info = - new ReadAsyncInfo(cb, cb_arg, clock_->NowMicros()); + + ReadAsyncInfo* read_async_info = new ReadAsyncInfo( + cb, cb_arg, (clock_ != nullptr ? clock_->NowMicros() : 0)); if (ShouldNotifyListeners()) { read_async_info->fs_start_ts_ = FileOperationInfo::StartNow(); @@ -495,16 +527,16 @@ IOStatus RandomAccessFileReader::ReadAsync( assert(read_async_info->buf_.CurrentSize() == 0); - StopWatch sw(clock_, nullptr /*stats*/, - Histograms::HISTOGRAM_ENUM_MAX /*hist_type*/, - Histograms::HISTOGRAM_ENUM_MAX, &elapsed, true /*overwrite*/, + StopWatch sw(clock_, stats_, hist_type_, + GetFileReadHistograms(stats_, opts.io_activity), + (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/, true /*delay_enabled*/); s = file_->ReadAsync(aligned_req, opts, read_async_callback, read_async_info, io_handle, del_fn, nullptr /*dbg*/); } else { - StopWatch sw(clock_, nullptr /*stats*/, - Histograms::HISTOGRAM_ENUM_MAX /*hist_type*/, - Histograms::HISTOGRAM_ENUM_MAX, &elapsed, true /*overwrite*/, + StopWatch sw(clock_, stats_, hist_type_, + GetFileReadHistograms(stats_, opts.io_activity), + (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/, true /*delay_enabled*/); s = file_->ReadAsync(req, opts, read_async_callback, read_async_info, io_handle, del_fn, nullptr /*dbg*/); diff --git a/file/random_access_file_reader.h b/file/random_access_file_reader.h index ab4d1e797..93cbe0e1a 100644 --- a/file/random_access_file_reader.h +++ b/file/random_access_file_reader.h @@ -164,31 +164,18 @@ class RandomAccessFileReader { // 2. Otherwise, scratch is not used and can be null, the aligned_buf owns // the internally allocated buffer on return, and the result refers to a // region in aligned_buf. - // - // `rate_limiter_priority` is used to charge the internal rate limiter when - // enabled. The special value `Env::IO_TOTAL` makes this operation bypass the - // rate limiter. IOStatus Read(const IOOptions& opts, uint64_t offset, size_t n, Slice* result, - char* scratch, AlignedBuf* aligned_buf, - Env::IOPriority rate_limiter_priority) const; + char* scratch, AlignedBuf* aligned_buf) const; // REQUIRES: // num_reqs > 0, reqs do not overlap, and offsets in reqs are increasing. // In non-direct IO mode, aligned_buf should be null; // In direct IO mode, aligned_buf stores the aligned buffer allocated inside // MultiRead, the result Slices in reqs refer to aligned_buf. - // - // `rate_limiter_priority` will be used to charge the internal rate limiter. - // It is not yet supported so the client must provide the special value - // `Env::IO_TOTAL` to bypass the rate limiter. IOStatus MultiRead(const IOOptions& opts, FSReadRequest* reqs, - size_t num_reqs, AlignedBuf* aligned_buf, - Env::IOPriority rate_limiter_priority) const; + size_t num_reqs, AlignedBuf* aligned_buf) const; - IOStatus Prefetch(uint64_t offset, size_t n, - const Env::IOPriority rate_limiter_priority) const { - IOOptions opts; - opts.rate_limiter_priority = rate_limiter_priority; + IOStatus Prefetch(const IOOptions& opts, uint64_t offset, size_t n) const { return file_->Prefetch(offset, n, opts, nullptr); } diff --git a/file/random_access_file_reader_test.cc b/file/random_access_file_reader_test.cc index 82ddcfff9..f081795b9 100644 --- a/file/random_access_file_reader_test.cc +++ b/file/random_access_file_reader_test.cc @@ -83,8 +83,9 @@ TEST_F(RandomAccessFileReaderTest, ReadDirectIO) { Slice result; AlignedBuf buf; for (Env::IOPriority rate_limiter_priority : {Env::IO_LOW, Env::IO_TOTAL}) { - ASSERT_OK(r->Read(IOOptions(), offset, len, &result, nullptr, &buf, - rate_limiter_priority)); + IOOptions io_opts; + io_opts.rate_limiter_priority = rate_limiter_priority; + ASSERT_OK(r->Read(io_opts, offset, len, &result, nullptr, &buf)); ASSERT_EQ(result.ToString(), content.substr(offset, len)); } } @@ -146,8 +147,8 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) { reqs.push_back(std::move(r0)); reqs.push_back(std::move(r1)); AlignedBuf aligned_buf; - ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf, - Env::IO_TOTAL /*rate_limiter_priority*/)); + ASSERT_OK( + r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf)); AssertResult(content, reqs); @@ -191,8 +192,8 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) { reqs.push_back(std::move(r1)); reqs.push_back(std::move(r2)); AlignedBuf aligned_buf; - ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf, - Env::IO_TOTAL /*rate_limiter_priority*/)); + ASSERT_OK( + r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf)); AssertResult(content, reqs); @@ -236,8 +237,8 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) { reqs.push_back(std::move(r1)); reqs.push_back(std::move(r2)); AlignedBuf aligned_buf; - ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf, - Env::IO_TOTAL /*rate_limiter_priority*/)); + ASSERT_OK( + r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf)); AssertResult(content, reqs); @@ -273,8 +274,8 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) { reqs.push_back(std::move(r0)); reqs.push_back(std::move(r1)); AlignedBuf aligned_buf; - ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf, - Env::IO_TOTAL /*rate_limiter_priority*/)); + ASSERT_OK( + r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf)); AssertResult(content, reqs); @@ -424,7 +425,9 @@ TEST(FSReadRequest, TryMerge) { src.scratch = nullptr; ASSERT_OK(src.status); - if (reverse) std::swap(dest, src); + if (reverse) { + std::swap(dest, src); + } ASSERT_TRUE(TryMerge(&dest, src)); ASSERT_EQ(dest.offset, 0); ASSERT_EQ(dest.len, 10); @@ -447,7 +450,9 @@ TEST(FSReadRequest, TryMerge) { src.scratch = nullptr; ASSERT_OK(src.status); - if (reverse) std::swap(dest, src); + if (reverse) { + std::swap(dest, src); + } ASSERT_TRUE(TryMerge(&dest, src)); ASSERT_EQ(dest.offset, 0); ASSERT_EQ(dest.len, 10); @@ -470,7 +475,9 @@ TEST(FSReadRequest, TryMerge) { src.scratch = nullptr; ASSERT_OK(src.status); - if (reverse) std::swap(dest, src); + if (reverse) { + std::swap(dest, src); + } ASSERT_TRUE(TryMerge(&dest, src)); ASSERT_EQ(dest.offset, 0); ASSERT_EQ(dest.len, 10); diff --git a/file/sequence_file_reader.h b/file/sequence_file_reader.h index 14350e8de..dc0e61bd2 100644 --- a/file/sequence_file_reader.h +++ b/file/sequence_file_reader.h @@ -99,6 +99,9 @@ class SequentialFileReader { // when less than n bytes are actually read (e.g. at end of file). To avoid // overcharging the rate limiter, the caller can use file size to cap n to // read until end of file. + // + // TODO(hx235): accept parameter `IOOptions` containing + // `rate_limiter_priority` like RandomAccessFileReader::Read() IOStatus Read(size_t n, Slice* result, char* scratch, Env::IOPriority rate_limiter_priority); diff --git a/include/rocksdb/advanced_cache.h b/include/rocksdb/advanced_cache.h index 997a41499..e2aefdd01 100644 --- a/include/rocksdb/advanced_cache.h +++ b/include/rocksdb/advanced_cache.h @@ -13,7 +13,9 @@ #include #include "rocksdb/cache.h" +#include "rocksdb/compression_type.h" #include "rocksdb/memory_allocator.h" +#include "rocksdb/options.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" @@ -66,7 +68,7 @@ class Cache { enum class Priority { HIGH, LOW, BOTTOM }; // A set of callbacks to allow objects in the primary block cache to be - // be persisted in a secondary cache. The purpose of the secondary cache + // persisted in a secondary cache. The purpose of the secondary cache // is to support other ways of caching the object, such as persistent or // compressed data, that may require the object to be parsed and transformed // in some way. Since the primary cache holds C++ objects and the secondary @@ -109,13 +111,18 @@ class Cache { // pointer into static data). using DeleterFn = void (*)(ObjectPtr obj, MemoryAllocator* allocator); - // The CreateCallback is takes in a buffer from the NVM cache and constructs - // an object using it. The callback doesn't have ownership of the buffer and + // The CreateCallback is takes in a buffer from the secondary cache and + // constructs an object using it. The buffer could be compressed or + // uncompressed, as indicated by the type argument. If compressed, + // the callback is responsible for uncompressing it using information + // from the context, such as compression dictionary. + // The callback doesn't have ownership of the buffer and // should copy the contents into its own buffer. The CreateContext* is // provided by Lookup and may be used to follow DB- or CF-specific settings. // In case of some error, non-OK is returned and the caller should ignore // any result in out_obj. (The implementation must clean up after itself.) - using CreateCallback = Status (*)(const Slice& data, CreateContext* context, + using CreateCallback = Status (*)(const Slice& data, CompressionType type, + CacheTier source, CreateContext* context, MemoryAllocator* allocator, ObjectPtr* out_obj, size_t* out_charge); @@ -242,12 +249,19 @@ class Cache { // the item is only inserted into the primary cache. It may // defer the insertion to the secondary cache as it sees fit. // + // Along with the object pointer, the caller may pass a Slice pointing to + // the compressed serialized data of the object. If compressed is + // non-empty, then the caller must pass the type indicating the compression + // algorithm used. The cache may, optionally, also insert the compressed + // block into one or more cache tiers. + // // When the inserted entry is no longer needed, it will be destroyed using // helper->del_cb (if non-nullptr). - virtual Status Insert(const Slice& key, ObjectPtr obj, - const CacheItemHelper* helper, size_t charge, - Handle** handle = nullptr, - Priority priority = Priority::LOW) = 0; + virtual Status Insert( + const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW, const Slice& compressed = Slice(), + CompressionType type = CompressionType::kNoCompression) = 0; // Similar to Insert, but used for creating cache entries that cannot // be found with Lookup, such as for memory charging purposes. The @@ -361,6 +375,14 @@ class Cache { // Returns the helper for the specified entry. virtual const CacheItemHelper* GetCacheItemHelper(Handle* handle) const = 0; + virtual Status GetSecondaryCacheCapacity(size_t& /*size*/) const { + return Status::NotSupported(); + } + + virtual Status GetSecondaryCachePinnedUsage(size_t& /*size*/) const { + return Status::NotSupported(); + } + // Call this on shutdown if you want to speed it up. Cache will disown // any underlying data and will not free it on delete. This call will leak // memory - call this only if you're shutting down the process. @@ -514,7 +536,8 @@ class Cache { // returns `true` if it has taken ownership of the Value (object), or // `false` if the cache should destroy it as usual. Regardless, Ref() and // Release() cannot be called on this Handle that is poised for eviction. - using EvictionCallback = std::function; + using EvictionCallback = + std::function; // Sets an eviction callback for this Cache. Not thread safe and only // supports being set once, so should only be used during initialization // or destruction, guaranteed before or after any thread-shared operations. @@ -535,11 +558,14 @@ class CacheWrapper : public Cache { // Only function that derived class must provide // const char* Name() const override { ... } - Status Insert(const Slice& key, ObjectPtr value, - const CacheItemHelper* helper, size_t charge, - Handle** handle = nullptr, - Priority priority = Priority::LOW) override { - return target_->Insert(key, value, helper, charge, handle, priority); + Status Insert( + const Slice& key, ObjectPtr value, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW, + const Slice& compressed_value = Slice(), + CompressionType type = CompressionType::kNoCompression) override { + return target_->Insert(key, value, helper, charge, handle, priority, + compressed_value, type); } Handle* CreateStandalone(const Slice& key, ObjectPtr obj, @@ -577,6 +603,14 @@ class CacheWrapper : public Cache { return target_->HasStrictCapacityLimit(); } + size_t GetOccupancyCount() const override { + return target_->GetOccupancyCount(); + } + + size_t GetTableAddressCount() const override { + return target_->GetTableAddressCount(); + } + size_t GetCapacity() const override { return target_->GetCapacity(); } size_t GetUsage() const override { return target_->GetUsage(); } @@ -612,6 +646,14 @@ class CacheWrapper : public Cache { target_->WaitAll(async_handles, count); } + uint32_t GetHashSeed() const override { return target_->GetHashSeed(); } + + void ReportProblems(const std::shared_ptr& info_log) const override { + target_->ReportProblems(info_log); + } + + const std::shared_ptr& GetTarget() { return target_; } + protected: std::shared_ptr target_; }; diff --git a/include/rocksdb/advanced_options.h b/include/rocksdb/advanced_options.h index df7bb4e32..e5ffe8944 100644 --- a/include/rocksdb/advanced_options.h +++ b/include/rocksdb/advanced_options.h @@ -76,9 +76,16 @@ struct CompressionOptions { // zlib only: windowBits parameter. See https://www.zlib.net/manual.html int window_bits = -14; - // Compression "level" applicable to zstd, zlib, LZ4. Except for + // Compression "level" applicable to zstd, zlib, LZ4, and LZ4HC. Except for // kDefaultCompressionLevel (see above), the meaning of each value depends - // on the compression algorithm. + // on the compression algorithm. Decreasing across non- + // `kDefaultCompressionLevel` values will either favor speed over + // compression ratio or have no effect. + // + // In LZ4 specifically, the absolute value of a negative `level` internally + // configures the `acceleration` parameter. For example, set `level=-10` for + // `acceleration=10`. This negation is necessary to ensure decreasing `level` + // values favor speed over compression ratio. int level = kDefaultCompressionLevel; // zlib only: strategy parameter. See https://www.zlib.net/manual.html @@ -181,6 +188,14 @@ struct CompressionOptions { // compressed by less than 12.5% (minimum ratio of 1.143:1). int max_compressed_bytes_per_kb = 1024 * 7 / 8; + // ZSTD only. + // Enable compression algorithm's checksum feature. + // (https://github.com/facebook/zstd/blob/d857369028d997c92ff1f1861a4d7f679a125464/lib/zstd.h#L428) + // Each compressed frame will have a 32-bit checksum attached. The checksum + // computed from the uncompressed data and can be verified during + // decompression. + bool checksum = false; + // A convenience function for setting max_compressed_bytes_per_kb based on a // minimum acceptable compression ratio (uncompressed size over compressed // size). @@ -260,7 +275,8 @@ struct CompactionOptionsFIFO { // In the future, we may add more caching layers. enum class CacheTier : uint8_t { kVolatileTier = 0, - kNonVolatileBlockTier = 0x01, + kVolatileCompressedTier = 0x01, + kNonVolatileBlockTier = 0x02, }; enum UpdateStatus { // Return status For inplace update callback @@ -698,7 +714,7 @@ struct AdvancedColumnFamilyOptions { // // Dynamically changeable through SetOptions() API std::vector max_bytes_for_level_multiplier_additional = - std::vector(num_levels, 1); + std::vector(static_cast(num_levels), 1); // We try to limit number of bytes in one compaction to be lower than this // threshold. But it's not guaranteed. @@ -869,6 +885,15 @@ struct AdvancedColumnFamilyOptions { // FIFO: Files with all keys older than TTL will be deleted. TTL is only // supported if option max_open_files is set to -1. // + // Universal: users should only set the option `periodic_compaction_seconds` + // below instead. For backward compatibility, this option has the same + // meaning as `periodic_compaction_seconds`. See more in comments for + // `periodic_compaction_seconds` on the interaction between these two + // options. + // + // This option only supports block based table format for any compaction + // style. + // // unit: seconds. Ex: 1 day = 1 * 24 * 60 * 60 // 0 means disabling. // UINT64_MAX - 1 (0xfffffffffffffffe) is special flag to allow RocksDB to @@ -877,10 +902,32 @@ struct AdvancedColumnFamilyOptions { // Default: 30 days if using block based table. 0 (disable) otherwise. // // Dynamically changeable through SetOptions() API + // Note that dynamically changing this option only works for leveled and FIFO + // compaction. For universal compaction, dynamically changing this option has + // no effect, users should dynamically change `periodic_compaction_seconds` + // instead. uint64_t ttl = 0xfffffffffffffffe; - // Files older than this value will be picked up for compaction, and - // re-written to the same level as they were before. + // This option has different meanings for different compaction styles: + // + // Leveled: files older than `periodic_compaction_seconds` will be picked up + // for compaction and will be re-written to the same level as they were + // before. + // + // FIFO: not supported. Setting this option has no effect for FIFO compaction. + // + // Universal: when there are files older than `periodic_compaction_seconds`, + // rocksdb will try to do as large a compaction as possible including the + // last level. Such compaction is only skipped if only last level is to + // be compacted and no file in last level is older than + // `periodic_compaction_seconds`. See more in + // UniversalCompactionBuilder::PickPeriodicCompaction(). + // For backward compatibility, the effective value of this option takes + // into account the value of option `ttl`. The logic is as follows: + // - both options are set to 30 days if they have the default value. + // - if both options are zero, zero is picked. Otherwise, we take the min + // value among non-zero options values (i.e. takes the stricter limit). + // // One main use of the feature is to make sure a file goes through compaction // filters periodically. Users can also use the feature to clear up SST // files using old format. @@ -890,19 +937,19 @@ struct AdvancedColumnFamilyOptions { // age is based on the file's last modified time (given by the underlying // Env). // - // Supported in leveled and universal compaction. - // In Universal compaction, rocksdb will try to do a full compaction when - // possible, see more in UniversalCompactionBuilder::PickPeriodicCompaction(). + // This option only supports block based table format for any compaction + // style. + // // unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60 // // Values: // 0: Turn off Periodic compactions. - // UINT64_MAX - 1 (i.e 0xfffffffffffffffe): Let RocksDB control this feature - // as needed. For now, RocksDB will change this value to 30 days - // (i.e 30 * 24 * 60 * 60) so that every file goes through the compaction - // process at least once every 30 days if not compacted sooner. + // UINT64_MAX - 1 (0xfffffffffffffffe) is special flag to allow RocksDB to + // pick default. // - // Default: UINT64_MAX - 1 (allow RocksDB to auto-tune) + // Default: 30 days if using block based table format + compaction filter + + // leveled compaction or block based table format + universal compaction. + // 0 (disabled) otherwise. // // Dynamically changeable through SetOptions() API uint64_t periodic_compaction_seconds = 0xfffffffffffffffe; @@ -929,6 +976,14 @@ struct AdvancedColumnFamilyOptions { Temperature bottommost_temperature = Temperature::kUnknown; Temperature last_level_temperature = Temperature::kUnknown; + // EXPERIMENTAL + // When this field is set, all SST files without an explicitly set temperature + // will be treated as if they have this temperature for file reading + // accounting purpose, such as io statistics, io perf context. + // + // Not dynamically changeable, change it requires db restart. + Temperature default_temperature = Temperature::kUnknown; + // EXPERIMENTAL // The feature is still in development and is incomplete. // If this option is set, when data insert time is within this time range, it @@ -1105,6 +1160,7 @@ struct AdvancedColumnFamilyOptions { // // Default: 0 (no protection) // Supported values: 0, 1, 2, 4, 8. + // Dynamically changeable through the SetOptions() API. uint32_t memtable_protection_bytes_per_key = 0; // UNDER CONSTRUCTION -- DO NOT USE @@ -1124,10 +1180,17 @@ struct AdvancedColumnFamilyOptions { // while set this flag to be `false`: user keys in the newly generated SST // files are of the same format as the existing SST files. // + // Currently only user comparator that formats user-defined timesamps as + // uint64_t via using one of the RocksDB provided comparator + // `ComparatorWithU64TsImpl` are supported. + // // When setting this flag to `false`, users should also call // `DB::IncreaseFullHistoryTsLow` to set a cutoff timestamp for flush. RocksDB // refrains from flushing a memtable with data still above - // the cutoff timestamp with best effort. Users can do user-defined + // the cutoff timestamp with best effort. If this cutoff timestamp is not set, + // flushing continues normally. + // + // Users can do user-defined // multi-versioned read above the cutoff timestamp. When users try to read // below the cutoff timestamp, an error will be returned. // @@ -1135,8 +1198,13 @@ struct AdvancedColumnFamilyOptions { // persisted to WAL even if this flag is set to `false`. The benefit of this // is that user-defined timestamps can be recovered with the caveat that users // should flush all memtables so there is no active WAL files before doing a - // downgrade or toggling on / off the user-defined timestamp feature on a - // column family. + // downgrade. In order to use WAL to recover user-defined timestamps, users of + // this feature would want to set both `avoid_flush_during_shutdown` and + // `avoid_flush_during_recovery` to be true. + // + // Note that setting this flag to false is not supported in combination with + // atomic flush, or concurrent memtable write enabled by + // `allow_concurrent_memtable_write`. // // Default: true (user-defined timestamps are persisted) // Not dynamically changeable, change it requires db restart and @@ -1155,8 +1223,21 @@ struct AdvancedColumnFamilyOptions { // // Default: 0 (no protection) // Supported values: 0, 1, 2, 4, 8. + // Dynamically changeable through the SetOptions() API. uint8_t block_protection_bytes_per_key = 0; + // For leveled compaction, RocksDB may compact a file at the bottommost level + // if it can compact away data that were protected by some snapshot. + // The compaction reason in LOG for this kind of compactions is + // "BottommostFiles". Usually such compaction can happen as soon as a + // relevant snapshot is released. This option allows user to delay + // such compactions. A file is qualified for "BottommostFiles" compaction + // if it is at least "bottommost_file_compaction_delay" seconds old. + // + // Default: 0 (no delay) + // Dynamically changeable through the SetOptions() API. + uint32_t bottommost_file_compaction_delay = 0; + // Create ColumnFamilyOptions with default values for all fields AdvancedColumnFamilyOptions(); // Create ColumnFamilyOptions from Options diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 407eb4720..8a26585fe 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -78,6 +78,7 @@ typedef struct rocksdb_lru_cache_options_t rocksdb_lru_cache_options_t; typedef struct rocksdb_hyper_clock_cache_options_t rocksdb_hyper_clock_cache_options_t; typedef struct rocksdb_cache_t rocksdb_cache_t; +typedef struct rocksdb_write_buffer_manager_t rocksdb_write_buffer_manager_t; typedef struct rocksdb_compactionfilter_t rocksdb_compactionfilter_t; typedef struct rocksdb_compactionfiltercontext_t rocksdb_compactionfiltercontext_t; @@ -136,6 +137,10 @@ typedef struct rocksdb_wal_iterator_t rocksdb_wal_iterator_t; typedef struct rocksdb_wal_readoptions_t rocksdb_wal_readoptions_t; typedef struct rocksdb_memory_consumers_t rocksdb_memory_consumers_t; typedef struct rocksdb_memory_usage_t rocksdb_memory_usage_t; +typedef struct rocksdb_statistics_histogram_data_t + rocksdb_statistics_histogram_data_t; +typedef struct rocksdb_wait_for_compact_options_t + rocksdb_wait_for_compact_options_t; /* DB operations */ @@ -1056,6 +1061,8 @@ rocksdb_block_based_options_set_pin_top_level_index_and_filter( rocksdb_block_based_table_options_t*, unsigned char); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory( rocksdb_options_t* opt, rocksdb_block_based_table_options_t* table_options); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_buffer_manager( + rocksdb_options_t* opt, rocksdb_write_buffer_manager_t* wbm); /* Cuckoo table options */ @@ -1139,6 +1146,8 @@ extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_paranoid_checks( rocksdb_options_t*); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_paths( rocksdb_options_t*, const rocksdb_dbpath_t** path_values, size_t num_paths); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_cf_paths( + rocksdb_options_t*, const rocksdb_dbpath_t** path_values, size_t num_paths); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_env(rocksdb_options_t*, rocksdb_env_t*); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log(rocksdb_options_t*, @@ -1252,6 +1261,26 @@ rocksdb_options_set_max_bytes_for_level_multiplier_additional( rocksdb_options_t*, int* level_values, size_t num_levels); extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_statistics( rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_periodic_compaction_seconds( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_periodic_compaction_seconds(rocksdb_options_t*); + +enum { + rocksdb_statistics_level_disable_all = 0, + rocksdb_statistics_level_except_tickers = + rocksdb_statistics_level_disable_all, + rocksdb_statistics_level_except_histogram_or_timers = 1, + rocksdb_statistics_level_except_timers = 2, + rocksdb_statistics_level_except_detailed_timers = 3, + rocksdb_statistics_level_except_time_for_mutex = 4, + rocksdb_statistics_level_all = 5, +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_statistics_level( + rocksdb_options_t*, int level); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_statistics_level( + rocksdb_options_t*); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, unsigned char val); @@ -1328,6 +1357,11 @@ extern ROCKSDB_LIBRARY_API int rocksdb_options_get_prepopulate_blob_cache( /* returns a pointer to a malloc()-ed, null terminated string */ extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string( rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_options_statistics_get_ticker_count( + rocksdb_options_t* opt, uint32_t ticker_type); +extern ROCKSDB_LIBRARY_API void rocksdb_options_statistics_get_histogram_data( + rocksdb_options_t* opt, uint32_t histogram_type, + rocksdb_statistics_histogram_data_t* const data); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_write_buffer_number( rocksdb_options_t*, int); @@ -1643,6 +1677,10 @@ extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_compression( /* RateLimiter */ extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* rocksdb_ratelimiter_create( int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness); +extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* +rocksdb_ratelimiter_create_auto_tuned(int64_t rate_bytes_per_sec, + int64_t refill_period_us, + int32_t fairness); extern ROCKSDB_LIBRARY_API void rocksdb_ratelimiter_destroy( rocksdb_ratelimiter_t*); @@ -1921,6 +1959,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_timestamp( rocksdb_readoptions_t*, const char* ts, size_t tslen); extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iter_start_ts( rocksdb_readoptions_t*, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_auto_readahead_size( + rocksdb_readoptions_t*, unsigned char); /* Write options */ @@ -2040,6 +2080,36 @@ rocksdb_cache_get_table_address_count(const rocksdb_cache_t* cache); extern ROCKSDB_LIBRARY_API size_t rocksdb_cache_get_occupancy_count(const rocksdb_cache_t* cache); +/* WriteBufferManager */ + +extern ROCKSDB_LIBRARY_API rocksdb_write_buffer_manager_t* +rocksdb_write_buffer_manager_create(size_t buffer_size, bool allow_stall); +extern ROCKSDB_LIBRARY_API rocksdb_write_buffer_manager_t* +rocksdb_write_buffer_manager_create_with_cache(size_t buffer_size, + const rocksdb_cache_t* cache, + bool allow_stall); + +extern ROCKSDB_LIBRARY_API void rocksdb_write_buffer_manager_destroy( + rocksdb_write_buffer_manager_t* wbm); +extern ROCKSDB_LIBRARY_API bool rocksdb_write_buffer_manager_enabled( + rocksdb_write_buffer_manager_t* wbm); +extern ROCKSDB_LIBRARY_API bool rocksdb_write_buffer_manager_cost_to_cache( + rocksdb_write_buffer_manager_t* wbm); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_write_buffer_manager_memory_usage(rocksdb_write_buffer_manager_t* wbm); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_write_buffer_manager_mutable_memtable_memory_usage( + rocksdb_write_buffer_manager_t* wbm); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_write_buffer_manager_dummy_entries_in_cache_usage( + rocksdb_write_buffer_manager_t* wbm); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_write_buffer_manager_buffer_size(rocksdb_write_buffer_manager_t* wbm); +extern ROCKSDB_LIBRARY_API void rocksdb_write_buffer_manager_set_buffer_size( + rocksdb_write_buffer_manager_t* wbm, size_t new_size); +extern ROCKSDB_LIBRARY_API void rocksdb_write_buffer_manager_set_allow_stall( + rocksdb_write_buffer_manager_t* wbm, bool new_allow_stall); + /* HyperClockCache */ extern ROCKSDB_LIBRARY_API rocksdb_hyper_clock_cache_options_t* @@ -2257,7 +2327,8 @@ extern ROCKSDB_LIBRARY_API rocksdb_fifo_compaction_options_t* rocksdb_fifo_compaction_options_create(void); extern ROCKSDB_LIBRARY_API void rocksdb_fifo_compaction_options_set_allow_compaction( - rocksdb_fifo_compaction_options_t* fifo_opts, unsigned char allow_compaction); + rocksdb_fifo_compaction_options_t* fifo_opts, + unsigned char allow_compaction); extern ROCKSDB_LIBRARY_API unsigned char rocksdb_fifo_compaction_options_get_allow_compaction( rocksdb_fifo_compaction_options_t* fifo_opts); @@ -2399,6 +2470,9 @@ extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_relative_filename( rocksdb_sst_file_metadata_t* file_meta); +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_directory( + rocksdb_sst_file_metadata_t* file_meta); + extern ROCKSDB_LIBRARY_API uint64_t rocksdb_sst_file_metadata_get_size(rocksdb_sst_file_metadata_t* file_meta); @@ -2880,6 +2954,67 @@ extern ROCKSDB_LIBRARY_API void rocksdb_disable_manual_compaction( extern ROCKSDB_LIBRARY_API void rocksdb_enable_manual_compaction(rocksdb_t* db); +extern ROCKSDB_LIBRARY_API rocksdb_statistics_histogram_data_t* +rocksdb_statistics_histogram_data_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_statistics_histogram_data_destroy( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_median( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_p95( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_p99( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_average( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_std_dev( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_max( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_statistics_histogram_data_get_count( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_statistics_histogram_data_get_sum( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_min( + rocksdb_statistics_histogram_data_t* data); + +extern ROCKSDB_LIBRARY_API void rocksdb_wait_for_compact( + rocksdb_t* db, rocksdb_wait_for_compact_options_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_wait_for_compact_options_t* +rocksdb_wait_for_compact_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_wait_for_compact_options_destroy( + rocksdb_wait_for_compact_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_wait_for_compact_options_set_abort_on_pause( + rocksdb_wait_for_compact_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_wait_for_compact_options_get_abort_on_pause( + rocksdb_wait_for_compact_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_wait_for_compact_options_set_flush( + rocksdb_wait_for_compact_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_wait_for_compact_options_get_flush( + rocksdb_wait_for_compact_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_wait_for_compact_options_set_close_db( + rocksdb_wait_for_compact_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_wait_for_compact_options_get_close_db( + rocksdb_wait_for_compact_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_wait_for_compact_options_set_timeout( + rocksdb_wait_for_compact_options_t* opt, uint64_t microseconds); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_wait_for_compact_options_get_timeout( + rocksdb_wait_for_compact_options_t* opt); + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/include/rocksdb/cache.h b/include/rocksdb/cache.h index 550b45859..2a358504e 100644 --- a/include/rocksdb/cache.h +++ b/include/rocksdb/cache.h @@ -12,6 +12,7 @@ #pragma once #include +#include #include #include @@ -375,19 +376,23 @@ inline std::shared_ptr NewCompressedSecondaryCache( // compatible with HyperClockCache. // * Requires an extra tuning parameter: see estimated_entry_charge below. // Similarly, substantially changing the capacity with SetCapacity could -// harm efficiency. +// harm efficiency. -> EXPERIMENTAL: the tuning parameter can be set to 0 +// to find the appropriate balance automatically. // * Cache priorities are less aggressively enforced, which could cause // cache dilution from long range scans (unless they use fill_cache=false). -// * Can be worse for small caches, because if almost all of a cache shard is -// pinned (more likely with non-partitioned filters), then CLOCK eviction -// becomes very CPU intensive. // // See internal cache/clock_cache.h for full description. struct HyperClockCacheOptions : public ShardedCacheOptions { - // The estimated average `charge` associated with cache entries. This is a - // critical configuration parameter for good performance from the hyper - // cache, because having a table size that is fixed at creation time greatly - // reduces the required synchronization between threads. + // The estimated average `charge` associated with cache entries. + // + // EXPERIMENTAL: the field can be set to 0 to size the table dynamically + // and automatically. See also min_avg_entry_charge. This feature requires + // platform support for lazy anonymous memory mappings (incl Linux, Windows). + // Performance is very similar to choosing the best configuration parameter. + // + // PRODUCTION-TESTED: This is a critical configuration parameter for good + // performance, because having a table size that is fixed at creation time + // greatly reduces the required synchronization between threads. // * If the estimate is substantially too low (e.g. less than half the true // average) then metadata space overhead with be substantially higher (e.g. // 200 bytes per entry rather than 100). With kFullChargeCacheMetadata, this @@ -416,6 +421,60 @@ struct HyperClockCacheOptions : public ShardedCacheOptions { // to estimate toward the lower side than the higher side. size_t estimated_entry_charge; + // EXPERIMENTAL: When estimated_entry_charge == 0, this parameter establishes + // a promised lower bound on the average charge of all entries in the table, + // which is roughly the average uncompressed SST block size of block cache + // entries, typically > 4KB. The default should generally suffice with almost + // no cost. (This option is ignored for estimated_entry_charge > 0.) + // + // More detail: The table for indexing cache entries will grow automatically + // as needed, but a hard upper bound on that size is needed at creation time. + // The reason is that a contiguous memory mapping for the maximum size is + // created, but memory pages are only mapped to physical (RSS) memory as + // needed. If the average charge of all entries in the table falls below + // this value, the table will operate below its full logical capacity (total + // memory usage) because it has reached its physical capacity for efficiently + // indexing entries. The hash table is never allowed to exceed a certain safe + // load factor for efficient Lookup, Insert, etc. + size_t min_avg_entry_charge = 450; + + // A tuning parameter to cap eviction CPU usage in a "thrashing" situation + // by allowing the memory capacity to be exceeded slightly as needed. The + // default setting should offer balanced protection against excessive CPU + // and memory usage under extreme stress conditions, with no effect on + // normal operation. Such stress conditions are proportionally more likely + // with small caches (10s of MB or less) vs. large caches (GB-scale). + // (NOTE: With the unusual setting of strict_capacity_limit=true, this + // parameter is ignored.) + // + // BACKGROUND: Without some kind of limiter, inserting into a CLOCK-based + // cache with no evictable entries (all "pinned") requires scanning the + // entire cache to determine that nothing can be evicted. (By contrast, + // LRU caches can determine no entries are evictable in O(1) time, but + // require more synchronization/coordination on that eviction metadata.) + // This aspect of a CLOCK cache can make a stressed situation worse by + // bogging down the CPU with repeated scans of the cache. And with + // strict_capacity_limit=false (normal setting), finding something evictable + // doesn't change the outcome of insertion: the entry is inserted anyway + // and the cache is allowed to exceed its target capacity if necessary. + // + // SOLUTION: Eviction is aborted upon seeing some number of pinned + // entries before evicting anything, or if the ratio of pinned to evicted + // is too high. This setting `eviction_effort_cap` essentially controls both + // that allowed initial number of pinned entries and the maximum allowed + // ratio. As the pinned size approaches the target cache capacity, roughly + // 1/eviction_effort_cap additional portion of the capacity might be kept + // in memory and evictable in order to keep CLOCK eviction reasonably + // performant. Under the default setting and high stress conditions, this + // memory overhead is around 3-5%. Under normal or even moderate stress + // conditions, the memory overhead is negligible to zero. + // + // A large value like 1000 offers some protection with essentially no + // memory overhead, while the minimum value of 1 could be useful for a + // small cache where roughly doubling in size under stress could be OK to + // keep operations very fast. + int eviction_effort_cap = 30; + HyperClockCacheOptions( size_t _capacity, size_t _estimated_entry_charge, int _num_shard_bits = -1, bool _strict_capacity_limit = false, @@ -447,18 +506,67 @@ enum PrimaryCacheType { kCacheTypeMax, }; +enum TieredAdmissionPolicy { + // Automatically select the admission policy + kAdmPolicyAuto, + // During promotion/demotion, first time insert a placeholder entry, second + // time insert the full entry if the placeholder is found, i.e insert on + // second hit + kAdmPolicyPlaceholder, + // Same as kAdmPolicyPlaceholder, but also if an entry in the primary cache + // was a hit, then force insert it into the compressed secondary cache + kAdmPolicyAllowCacheHits, + // An admission policy for three cache tiers - primary uncompressed, + // compressed secondary, and a compressed local flash (non-volatile) cache. + // Each tier is managed as an independent queue. + kAdmPolicyThreeQueue, + kAdmPolicyMax, +}; + +// EXPERIMENTAL +// The following feature is experimental, and the API is subject to change +// // A 2-tier cache with a primary block cache, and a compressed secondary // cache. The returned cache instance will internally allocate a primary // uncompressed cache of the specified type, and a compressed secondary // cache. Any cache memory reservations, such as WriteBufferManager // allocations costed to the block cache, will be distributed // proportionally across both the primary and secondary. -struct TieredVolatileCacheOptions { - ShardedCacheOptions* cache_opts; - PrimaryCacheType cache_type; +struct TieredCacheOptions { + // This should point to an instance of either LRUCacheOptions or + // HyperClockCacheOptions, depending on the cache_type. In either + // case, the capacity and secondary_cache fields in those options + // should not be set. If set, they will be ignored by NewTieredCache. + ShardedCacheOptions* cache_opts = nullptr; + PrimaryCacheType cache_type = PrimaryCacheType::kCacheTypeLRU; + TieredAdmissionPolicy adm_policy = TieredAdmissionPolicy::kAdmPolicyAuto; CompressedSecondaryCacheOptions comp_cache_opts; + // Any capacity specified in LRUCacheOptions, HyperClockCacheOptions and + // CompressedSecondaryCacheOptions is ignored + // The total_capacity specified here is taken as the memory budget and + // divided between the primary block cache and compressed secondary cache + size_t total_capacity = 0; + double compressed_secondary_ratio = 0.0; + // An optional secondary cache that will serve as the persistent cache + // tier. If present, compressed blocks will be written to this + // secondary cache. + std::shared_ptr nvm_sec_cache; }; -extern std::shared_ptr NewTieredVolatileCache( - TieredVolatileCacheOptions& cache_opts); +extern std::shared_ptr NewTieredCache( + const TieredCacheOptions& cache_opts); + +// EXPERIMENTAL +// Dynamically update some of the parameters of a TieredCache. The input +// cache shared_ptr should have been allocated using NewTieredVolatileCache. +// At the moment, there are a couple of limitations - +// 1. The total_capacity should be > the WriteBufferManager max size, if +// using the block cache charging feature +// 2. Once the compressed secondary cache is disabled by setting the +// compressed_secondary_ratio to 0.0, it cannot be dynamically re-enabled +// again +extern Status UpdateTieredCache( + const std::shared_ptr& cache, int64_t total_capacity = -1, + double compressed_secondary_ratio = std::numeric_limits::max(), + TieredAdmissionPolicy adm_policy = TieredAdmissionPolicy::kAdmPolicyMax); } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/compaction_filter.h b/include/rocksdb/compaction_filter.h index b1b511613..1784f2329 100644 --- a/include/rocksdb/compaction_filter.h +++ b/include/rocksdb/compaction_filter.h @@ -16,6 +16,7 @@ #include "rocksdb/customizable.h" #include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/table_properties.h" #include "rocksdb/types.h" #include "rocksdb/wide_columns.h" @@ -160,10 +161,20 @@ class CompactionFilter : public Customizable { // Whether this table file is created as part of a compaction requested by // the client. bool is_manual_compaction; + // The lowest level among all the input files (if any) used in table + // creation + int input_start_level = kUnknownStartLevel; // The column family that will contain the created table file. uint32_t column_family_id; // Reason this table file is being created. TableFileCreationReason reason; + // Map from all the input files (if any) used in table creation to their + // table properties. When there are such input files but RocksDB fail to + // load their table properties, `input_table_properties` will be an empty + // map. + TablePropertiesCollection input_table_properties; + + static const int kUnknownStartLevel = -1; }; virtual ~CompactionFilter() {} diff --git a/include/rocksdb/compaction_job_stats.h b/include/rocksdb/compaction_job_stats.h index 5ff8eccc8..7e8153044 100644 --- a/include/rocksdb/compaction_job_stats.h +++ b/include/rocksdb/compaction_job_stats.h @@ -24,6 +24,9 @@ struct CompactionJobStats { // the elapsed CPU time of this compaction in microseconds. uint64_t cpu_micros; + // Used internally indicating whether a subcompaction's + // `num_input_records` is accurate. + bool has_num_input_records; // the number of compaction input records. uint64_t num_input_records; // the number of blobs read from blob files diff --git a/include/rocksdb/comparator.h b/include/rocksdb/comparator.h index ad1e71a11..4b39a2585 100644 --- a/include/rocksdb/comparator.h +++ b/include/rocksdb/comparator.h @@ -155,10 +155,50 @@ class Comparator : public Customizable, public CompareInterface { // Return a builtin comparator that uses lexicographic byte-wise // ordering. The result remains the property of this module and // must not be deleted. -extern const Comparator* BytewiseComparator(); +const Comparator* BytewiseComparator(); // Return a builtin comparator that uses reverse lexicographic byte-wise // ordering. -extern const Comparator* ReverseBytewiseComparator(); +const Comparator* ReverseBytewiseComparator(); + +// Returns a builtin comparator that enables user-defined timestamps (formatted +// as uint64_t) while ordering the user key part without UDT with a +// BytewiseComparator. +// For the same user key with different timestamps, larger (newer) timestamp +// comes first. +const Comparator* BytewiseComparatorWithU64Ts(); + +// Returns a builtin comparator that enables user-defined timestamps (formatted +// as uint64_t) while ordering the user key part without UDT with a +// ReverseBytewiseComparator. +// For the same user key with different timestamps, larger (newer) timestamp +// comes first. +const Comparator* ReverseBytewiseComparatorWithU64Ts(); + +// Decode a `U64Ts` timestamp returned by RocksDB to uint64_t. +// When a column family enables user-defined timestamp feature +// with `BytewiseComparatorWithU64Ts` or `ReverseBytewiseComparatorWithU64Ts` +// comparator, the `Iterator::timestamp()` API returns timestamp in `Slice` +// format. This util function helps to translate that `Slice` into an uint64_t +// type. +Status DecodeU64Ts(const Slice& ts, uint64_t* int_ts); + +// Encode an uint64_t timestamp into a U64Ts `Slice`, to be used as +// `ReadOptions.timestamp` for a column family that enables user-defined +// timestamp feature with `BytewiseComparatorWithU64Ts` or +// `ReverseBytewiseComparatorWithU64Ts` comparator. +// Be mindful that the returned `Slice` is backed by `ts_buf`. When `ts_buf` +// is deconstructed, the returned `Slice` can no longer be used. +Slice EncodeU64Ts(uint64_t ts, std::string* ts_buf); + +// Returns a `Slice` representing the maximum U64Ts timestamp. +// The returned `Slice` is backed by some static storage, so it's valid until +// program destruction. +Slice MaxU64Ts(); + +// Returns a `Slice` representing the minimum U64Ts timestamp. +// The returned `Slice` is backed by some static storage, so it's valid until +// program destruction. +Slice MinU64Ts(); } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h index 7ce676df0..cff03f2bc 100644 --- a/include/rocksdb/convenience.h +++ b/include/rocksdb/convenience.h @@ -459,7 +459,7 @@ Status VerifySstFileChecksum(const Options& options, // Verify the checksum of file Status VerifySstFileChecksum(const Options& options, const EnvOptions& env_options, - const ReadOptions& read_options, + const ReadOptions& _read_options, const std::string& file_path, const SequenceNumber& largest_seqno = 0); diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 436b430f1..5ae73182b 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -100,6 +100,8 @@ static const int kMinorVersion = __ROCKSDB_MINOR__; // A range of keys struct Range { + // In case of user_defined timestamp, if enabled, `start` and `limit` should + // point to key without timestamp part. Slice start; Slice limit; @@ -108,6 +110,8 @@ struct Range { }; struct RangePtr { + // In case of user_defined timestamp, if enabled, `start` and `limit` should + // point to key without timestamp part. const Slice* start; const Slice* limit; @@ -131,6 +135,11 @@ struct IngestExternalFileArg { }; struct GetMergeOperandsOptions { + // A limit on the number of merge operands returned by the GetMergeOperands() + // API. In contrast with ReadOptions::merge_operator_max_count, this is a hard + // limit: when it is exceeded, no merge operands will be returned and the + // query will fail with an Incomplete status. See also the + // DB::GetMergeOperands() API below. int expected_max_number_of_operands = 0; }; @@ -322,12 +331,17 @@ class DB { // If syncing is required, the caller must first call SyncWAL(), or Write() // using an empty write batch with WriteOptions.sync=true. // Regardless of the return status, the DB must be freed. + // // If the return status is Aborted(), closing fails because there is // unreleased snapshot in the system. In this case, users can release // the unreleased snapshots and try again and expect it to succeed. For // other status, re-calling Close() will be no-op and return the original // close status. If the return status is NotSupported(), then the DB // implementation does cleanup in the destructor + // + // WaitForCompact() with WaitForCompactOptions.close_db=true will be a good + // choice for users who want to wait for background work before closing + // (rather than aborting and potentially redoing some work on re-open) virtual Status Close() { return Status::NotSupported(); } // ListColumnFamilies will open the DB specified by argument name @@ -348,6 +362,10 @@ class DB { // Create a column_family and return the handle of column family // through the argument handle. + // NOTE: creating many column families one-by-one is not recommended because + // of quadratic overheads, such as writing a full OPTIONS file for all CFs + // after each new CF creation. Use CreateColumnFamilies(), or DB::Open() with + // create_missing_column_families=true. virtual Status CreateColumnFamily(const ColumnFamilyOptions& options, const std::string& column_family_name, ColumnFamilyHandle** handle); @@ -417,6 +435,10 @@ class DB { virtual Status PutEntity(const WriteOptions& options, ColumnFamilyHandle* column_family, const Slice& key, const WideColumns& columns); + // Split and store wide column entities in multiple column families (a.k.a. + // AttributeGroups) + virtual Status PutEntity(const WriteOptions& options, const Slice& key, + const AttributeGroups& attribute_groups); // Remove the database entry (if any) for "key". Returns OK on // success, and a non-OK status on error. It is not an error if "key" @@ -487,6 +509,15 @@ class DB { ColumnFamilyHandle* column_family, const Slice& begin_key, const Slice& end_key, const Slice& ts); + virtual Status DeleteRange(const WriteOptions& options, + const Slice& begin_key, const Slice& end_key) { + return DeleteRange(options, DefaultColumnFamily(), begin_key, end_key); + } + virtual Status DeleteRange(const WriteOptions& options, + const Slice& begin_key, const Slice& end_key, + const Slice& ts) { + return DeleteRange(options, DefaultColumnFamily(), begin_key, end_key, ts); + } // Merge the database entry for "key" with "value". Returns OK on success, // and a non-OK status on error. The semantics of this operation is @@ -584,6 +615,16 @@ class DB { return Status::NotSupported("GetEntity not supported"); } + // Returns logically grouped wide-column entities per column family (a.k.a. + // attribute groups) for a single key. PinnableAttributeGroups is a vector of + // PinnableAttributeGroup. Each PinnableAttributeGroup will have + // ColumnFamilyHandle* as input, and Status and PinnableWideColumns as output. + virtual Status GetEntity(const ReadOptions& /* options */, + const Slice& /* key */, + PinnableAttributeGroups* /* result */) { + return Status::NotSupported("GetEntity not supported"); + } + // Populates the `merge_operands` array with all the merge operands in the DB // for `key`. The `merge_operands` array will be populated in the order of // insertion. The number of entries populated in `merge_operands` will be @@ -844,6 +885,34 @@ class DB { } } + // Batched MultiGet-like API that returns attribute groups. + // An "attribute group" refers to a logical grouping of wide-column entities + // within RocksDB. These attribute groups are implemented using column + // families. Attribute group allows users to group wide-columns based on + // various criteria, such as similar access patterns or data types + // + // The input is a list of keys and PinnableAttributeGroups. For any given + // keys[i] (where 0 <= i < num_keys), results[i] will contain result for the + // ith key. Each result will be returned as PinnableAttributeGroups. + // PinnableAttributeGroups is a vector of PinnableAttributeGroup. Each + // PinnableAttributeGroup will contain a ColumnFamilyHandle pointer, Status + // and PinnableWideColumns. + // + // Note that it is the caller's responsibility to ensure that + // "keys" and "results" have the same "num_keys" number of objects. Also + // PinnableAttributeGroup needs to have ColumnFamilyHandle pointer set + // properly to get the corresponding wide columns from the column family. + virtual void MultiGetEntity(const ReadOptions& /* options */, size_t num_keys, + const Slice* /* keys */, + PinnableAttributeGroups* results) { + for (size_t i = 0; i < num_keys; ++i) { + for (size_t j = 0; j < results[i].size(); ++j) { + results[i][j].SetStatus( + Status::NotSupported("MultiGetEntity not supported")); + } + } + } + // If the key definitely does not exist in the database, then this method // returns false, else true. If the caller wants to obtain value when the key // is found in memory, a bool for 'value_found' must be passed. 'value_found' @@ -1352,6 +1421,9 @@ class DB { // the files. In this case, client could set options.change_level to true, to // move the files back to the minimum level capable of holding the data set // or a given level (specified by non-negative options.target_level). + // + // In case of user-defined timestamp, if enabled, `begin` and `end` should + // not contain timestamp. virtual Status CompactRange(const CompactRangeOptions& options, ColumnFamilyHandle* column_family, const Slice* begin, const Slice* end) = 0; @@ -1465,7 +1537,8 @@ class DB { // NOTE: This may also never return if there's sufficient ongoing writes that // keeps flush and compaction going without stopping. The user would have to // cease all the writes to DB to make this eventually return in a stable - // state. + // state. The user may also use timeout option in WaitForCompactOptions to + // make this stop waiting and return when timeout expires. virtual Status WaitForCompact( const WaitForCompactOptions& /* wait_for_compact_options */) = 0; @@ -1586,7 +1659,17 @@ class DB { virtual Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family, std::string* ts_low) = 0; - // Allow compactions to delete obsolete files. + // Enable deleting obsolete files. + // Usually users should only need to call this if they have previously called + // `DisableFileDeletions`. + // File deletions disabling and enabling is not controlled by a binary flag, + // instead it's represented as a counter to allow different callers to + // independently disable file deletion. Disabling file deletion can be + // critical for operations like making a backup. So the counter implementation + // makes the file deletion disabled as long as there is one caller requesting + // so, and only when every caller agrees to re-enable file deletion, it will + // be enabled. So be careful when calling this function with force = true as + // explained below. // If force == true, the call to EnableFileDeletions() will guarantee that // file deletions are enabled after the call, even if DisableFileDeletions() // was called multiple times before. @@ -1595,7 +1678,7 @@ class DB { // enabling the two methods to be called by two threads concurrently without // synchronization -- i.e., file deletions will be enabled only after both // threads call EnableFileDeletions() - virtual Status EnableFileDeletions(bool force = true) = 0; + virtual Status EnableFileDeletions(bool force) = 0; // Retrieves the creation time of the oldest file in the DB. // This API only works if max_open_files = -1, if it is not then @@ -1828,7 +1911,6 @@ class DB { virtual Status VerifyChecksum() { return VerifyChecksum(ReadOptions()); } - // Returns the unique ID which is read from IDENTITY file during the opening // of database by setting in the identity variable // Returns Status::OK if identity could be set properly @@ -1844,7 +1926,6 @@ class DB { // Returns default column family handle virtual ColumnFamilyHandle* DefaultColumnFamily() const = 0; - virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family, TablePropertiesCollection* props) = 0; virtual Status GetPropertiesOfAllTables(TablePropertiesCollection* props) { @@ -1910,7 +1991,6 @@ class DB { return Status::NotSupported("NewDefaultReplayer() is not implemented."); } - // Needed for StackableDB virtual DB* GetRootDB() { return this; } @@ -2010,5 +2090,4 @@ Status RepairDB(const std::string& dbname, const DBOptions& db_options, // families encountered during the repair Status RepairDB(const std::string& dbname, const Options& options); - } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/env.h b/include/rocksdb/env.h index ea99a7a9e..7b0220635 100644 --- a/include/rocksdb/env.h +++ b/include/rocksdb/env.h @@ -280,7 +280,7 @@ class Env : public Customizable { const EnvOptions& options); // Open `fname` for random read and write, if file doesn't exist the file - // will be created. On success, stores a pointer to the new file in + // will not be created. On success, stores a pointer to the new file in // *result and returns OK. On failure returns non-OK. // // The returned file will only be accessed by one thread at a time. @@ -441,6 +441,13 @@ class Env : public Customizable { kFlush = 0, kCompaction = 1, kDBOpen = 2, + kGet = 3, + kMultiGet = 4, + kDBIterator = 5, + kVerifyDBChecksum = 6, + kVerifyFileChecksums = 7, + kGetEntity = 8, + kMultiGetEntity = 9, kUnknown, // Keep last for easy array of non-unknowns }; diff --git a/include/rocksdb/file_system.h b/include/rocksdb/file_system.h index f8e321417..647aad6c9 100644 --- a/include/rocksdb/file_system.h +++ b/include/rocksdb/file_system.h @@ -120,6 +120,7 @@ struct IOOptions { // directories and list only files in GetChildren API. bool do_not_recurse; + // EXPERIMENTAL Env::IOActivity io_activity = Env::IOActivity::kUnknown; IOOptions() : IOOptions(false) {} @@ -390,7 +391,7 @@ class FileSystem : public Customizable { IODebugContext* dbg); // Open `fname` for random read and write, if file doesn't exist the file - // will be created. On success, stores a pointer to the new file in + // will not be created. On success, stores a pointer to the new file in // *result and returns OK. On failure returns non-OK. // // The returned file will only be accessed by one thread at a time. diff --git a/include/rocksdb/filter_policy.h b/include/rocksdb/filter_policy.h index 954d15b4a..039b826de 100644 --- a/include/rocksdb/filter_policy.h +++ b/include/rocksdb/filter_policy.h @@ -162,7 +162,7 @@ class FilterPolicy : public Customizable { // ignores trailing spaces, it would be incorrect to use a // FilterPolicy (like NewBloomFilterPolicy) that does not ignore // trailing spaces in keys. -extern const FilterPolicy* NewBloomFilterPolicy( +const FilterPolicy* NewBloomFilterPolicy( double bits_per_key, bool IGNORED_use_block_based_builder = false); // A new Bloom alternative that saves about 30% space compared to @@ -184,6 +184,11 @@ extern const FilterPolicy* NewBloomFilterPolicy( // flushes under Level and Universal compaction styles. // bloom_before_level=-1 -> Always generate Ribbon filters (except in // some extreme or exceptional cases). +// bloom_before_level=INT_MAX -> Always generate Bloom filters. +// +// The bloom_before_level option is mutable in the Configurable interface +// and through the SetOptions() API, as in +// db->SetOptions({{"table_factory.filter_policy.bloom_before_level", "3"}}); // // Ribbon filters are compatible with RocksDB >= 6.15.0. Earlier // versions reading the data will behave as if no filter was used @@ -200,7 +205,7 @@ extern const FilterPolicy* NewBloomFilterPolicy( // // Also consider using optimize_filters_for_memory to save filter // memory. -extern const FilterPolicy* NewRibbonFilterPolicy( - double bloom_equivalent_bits_per_key, int bloom_before_level = 0); +FilterPolicy* NewRibbonFilterPolicy(double bloom_equivalent_bits_per_key, + int bloom_before_level = 0); } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/iterator.h b/include/rocksdb/iterator.h index 9d4c9f73a..8568dd258 100644 --- a/include/rocksdb/iterator.h +++ b/include/rocksdb/iterator.h @@ -107,10 +107,16 @@ class Iterator : public Cleanable { // satisfied without doing some IO, then this returns Status::Incomplete(). virtual Status status() const = 0; - // If supported, renew the iterator to represent the latest state. The - // iterator will be invalidated after the call. Not supported if - // ReadOptions.snapshot is given when creating the iterator. - virtual Status Refresh() { + // If supported, the DB state that the iterator reads from is updated to + // the latest state. The iterator will be invalidated after the call. + // Regardless of whether the iterator was created/refreshed previously + // with or without a snapshot, the iterator will be reading the + // latest DB state after this call. + virtual Status Refresh() { return Refresh(nullptr); } + + // Similar to Refresh() but the iterator will be reading the latest DB state + // under the given snapshot. + virtual Status Refresh(const class Snapshot*) { return Status::NotSupported("Refresh() is not supported"); } @@ -127,6 +133,16 @@ class Iterator : public Cleanable { // Property "rocksdb.iterator.internal-key": // Get the user-key portion of the internal key at which the iteration // stopped. + // Property "rocksdb.iterator.write-time": + // DO NOT USE, UNDER CONSTRUCTION + // Get the unix time of the best estimate of the write time of the entry. + // Returned as 64-bit raw value (8 bytes). It can be converted to uint64_t + // with util method `DecodeU64Ts`. The accuracy of the write time depends on + // settings like preserve_internal_time_seconds. If this feature is + // disabled, this property will always be empty. The actual write time of + // the entry should be the same or newer than the returned write time. So + // this property can be interpreted as the possible oldest write time for + // the entry. virtual Status GetProperty(std::string prop_name, std::string* prop); virtual Slice timestamp() const { diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 87bc67869..2cc30d871 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -161,6 +161,9 @@ enum class CompactionReason : int { kNumOfReasons, }; +const char* GetCompactionReasonString(CompactionReason compaction_reason); + +// When adding flush reason, make sure to also update `GetFlushReasonString()`. enum class FlushReason : int { kOthers = 0x00, kGetLiveFiles = 0x01, @@ -178,8 +181,12 @@ enum class FlushReason : int { // will not be called to avoid many small immutable memtables. kErrorRecoveryRetryFlush = 0xc, kWalFull = 0xd, + // SwitchMemtable will not be called for this flush reason. + kCatchUpAfterErrorRecovery = 0xe, }; +const char* GetFlushReasonString(FlushReason flush_reason); + // TODO: In the future, BackgroundErrorReason will only be used to indicate // why the BG Error is happening (e.g., flush, compaction). We may introduce // other data structure to indicate other essential information such as diff --git a/include/rocksdb/memory_allocator.h b/include/rocksdb/memory_allocator.h index d126abfe6..dc744d7d1 100644 --- a/include/rocksdb/memory_allocator.h +++ b/include/rocksdb/memory_allocator.h @@ -81,7 +81,7 @@ struct JemallocAllocatorOptions { // The tcache normally incurs 0.5M extra memory usage per-thread. The usage // can be reduced by limiting allocation sizes to cache. extern Status NewJemallocNodumpAllocator( - JemallocAllocatorOptions& options, + const JemallocAllocatorOptions& options, std::shared_ptr* memory_allocator); } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/merge_operator.h b/include/rocksdb/merge_operator.h index 077130475..6be9e3962 100644 --- a/include/rocksdb/merge_operator.h +++ b/include/rocksdb/merge_operator.h @@ -8,10 +8,13 @@ #include #include #include +#include +#include #include #include "rocksdb/customizable.h" #include "rocksdb/slice.h" +#include "rocksdb/wide_columns.h" namespace ROCKSDB_NAMESPACE { @@ -33,7 +36,7 @@ class Logger; // into rocksdb); numeric addition and string concatenation are examples; // // b) MergeOperator - the generic class for all the more abstract / complex -// operations; one method (FullMergeV2) to merge a Put/Delete value with a +// operations; one method (FullMergeV3) to merge a Put/Delete value with a // merge operand; and another method (PartialMerge) that merges multiple // operands together. this is especially useful if your key values have // complex structures but you would still like to support client-specific @@ -158,6 +161,54 @@ class MergeOperator : public Customizable { virtual bool FullMergeV2(const MergeOperationInput& merge_in, MergeOperationOutput* merge_out) const; + struct MergeOperationInputV3 { + using ExistingValue = std::variant; + using OperandList = std::vector; + + explicit MergeOperationInputV3(const Slice& _key, + ExistingValue&& _existing_value, + const OperandList& _operand_list, + Logger* _logger) + : key(_key), + existing_value(std::move(_existing_value)), + operand_list(_operand_list), + logger(_logger) {} + + // The user key, including the user-defined timestamp if applicable. + const Slice& key; + // The base value of the merge operation. Can be one of three things (see + // the ExistingValue variant above): no existing value, plain existing + // value, or wide-column existing value. + ExistingValue existing_value; + // The list of operands to apply. + const OperandList& operand_list; + // The logger to use in case a failure happens during the merge operation. + Logger* logger; + }; + + struct MergeOperationOutputV3 { + using NewColumns = std::vector>; + using NewValue = std::variant; + + // The result of the merge operation. Can be one of three things (see the + // NewValue variant above): a new plain value, a new wide-column value, or + // an existing merge operand. + NewValue new_value; + // The scope of the failure if applicable. See above for more details. + OpFailureScope op_failure_scope = OpFailureScope::kDefault; + }; + + // An extended version of FullMergeV2() that supports wide columns on both the + // input and the output side, enabling the application to perform general + // transformations during merges. For backward compatibility, the default + // implementation calls FullMergeV2(). Specifically, if there is no base value + // or the base value is a plain key-value, the default implementation falls + // back to FullMergeV2(). If the base value is a wide-column entity, the + // default implementation invokes FullMergeV2() to perform the merge on the + // default column, and leaves any other columns unchanged. + virtual bool FullMergeV3(const MergeOperationInputV3& merge_in, + MergeOperationOutputV3* merge_out) const; + // This function performs merge(left_op, right_op) // when both the operands are themselves merge operation types // that you would have passed to a DB::Merge() call in the same order @@ -186,7 +237,7 @@ class MergeOperator : public Customizable { // TODO: Presently there is no way to differentiate between error/corruption // and simply "return false". For now, the client should simply return // false in any case it cannot perform partial-merge, regardless of reason. - // If there is corruption in the data, handle it in the FullMergeV2() function + // If there is corruption in the data, handle it in the FullMergeV3() function // and return false there. The default implementation of PartialMerge will // always return false. virtual bool PartialMerge(const Slice& /*key*/, const Slice& /*left_operand*/, @@ -243,8 +294,8 @@ class MergeOperator : public Customizable { // Doesn't help with iterators. // // Note: the merge operands are passed to this function in the reversed order - // relative to how they were merged (passed to FullMerge or FullMergeV2) - // for performance reasons, see also: + // relative to how they were merged (passed to + // FullMerge/FullMergeV2/FullMergeV3) for performance reasons, see also: // https://github.com/facebook/rocksdb/issues/3865 virtual bool ShouldMerge(const std::vector& /*operands*/) const { return false; diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 6cf91a491..ae5ed2c26 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -206,6 +207,7 @@ struct ColumnFamilyOptions : public AdvancedColumnFamilyOptions { // - kZSTD: 3 // - kZlibCompression: Z_DEFAULT_COMPRESSION (currently -1) // - kLZ4HCCompression: 0 + // - kLZ4: -1 (i.e., `acceleration=1`; see `CompressionOptions::level` doc) // - For all others, we do not specify a compression level // // Dynamically changeable through SetOptions() API @@ -331,6 +333,17 @@ struct ColumnFamilyOptions : public AdvancedColumnFamilyOptions { // Default: nullptr std::shared_ptr sst_partitioner_factory = nullptr; + // RocksDB will try to flush the current memtable after the number of range + // deletions is >= this limit. For workloads with many range + // deletions, limiting the number of range deletions in memtable can help + // prevent performance degradation and/or OOM caused by too many range + // tombstones in a single memtable. + // + // Default: 0 (disabled) + // + // Dynamically changeable through SetOptions() API + uint32_t memtable_max_range_deletions = 0; + // Create ColumnFamilyOptions with default values for all fields ColumnFamilyOptions(); // Create ColumnFamilyOptions from Options @@ -472,7 +485,8 @@ struct DBOptions { // Default: false bool create_if_missing = false; - // If true, missing column families will be automatically created. + // If true, missing column families will be automatically created on + // DB::Open(). // Default: false bool create_missing_column_families = false; @@ -490,11 +504,29 @@ struct DBOptions { // If true, during memtable flush, RocksDB will validate total entries // read in flush, and compare with counter inserted into it. + // // The option is here to turn the feature off in case this new validation - // feature has a bug. + // feature has a bug. The option may be removed in the future once the + // feature is stable. + // // Default: true bool flush_verify_memtable_count = true; + // If true, during compaction, RocksDB will count the number of entries + // read and compare it against the number of entries in the compaction + // input files. This is intended to add protection against corruption + // during compaction. Note that + // - this verification is not done for compactions during which a compaction + // filter returns kRemoveAndSkipUntil, and + // - the number of range deletions is not verified. + // + // The option is here to turn the feature off in case this new validation + // feature has a bug. The option may be removed in the future once the + // feature is stable. + // + // Default: true + bool compaction_verify_record_count = true; + // If true, the log numbers and sizes of the synced WALs are tracked // in MANIFEST. During DB recovery, if a synced WAL is missing // from disk, or the WAL's size does not match the recorded size in @@ -782,18 +814,23 @@ struct DBOptions { // Number of shards used for table cache. int table_cache_numshardbits = 6; - // The following two fields affect how archived logs will be deleted. - // 1. If both set to 0, logs will be deleted asap and will not get into - // the archive. - // 2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, - // WAL files will be checked every 10 min and if total size is greater - // then WAL_size_limit_MB, they will be deleted starting with the - // earliest until size_limit is met. All empty files will be deleted. - // 3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then - // WAL files will be checked every WAL_ttl_seconds / 2 and those that - // are older than WAL_ttl_seconds will be deleted. - // 4. If both are not 0, WAL files will be checked every 10 min and both - // checks will be performed with ttl being first. + // The following two fields affect when WALs will be archived and deleted. + // + // When both are zero, obsolete WALs will not be archived and will be deleted + // immediately. Otherwise, obsolete WALs will be archived prior to deletion. + // + // When `WAL_size_limit_MB` is nonzero, archived WALs starting with the + // earliest will be deleted until the total size of the archive falls below + // this limit. All empty WALs will be deleted. + // + // When `WAL_ttl_seconds` is nonzero, archived WALs older than + // `WAL_ttl_seconds` will be deleted. + // + // When only `WAL_ttl_seconds` is nonzero, the frequency at which archived + // WALs are deleted is every `WAL_ttl_seconds / 2` seconds. When only + // `WAL_size_limit_MB` is nonzero, the deletion frequency is every ten + // minutes. When both are nonzero, the deletion frequency is the minimum of + // those two values. uint64_t WAL_ttl_seconds = 0; uint64_t WAL_size_limit_MB = 0; @@ -913,6 +950,9 @@ struct DBOptions { // Default: null std::shared_ptr write_buffer_manager = nullptr; + // DEPRECATED + // This flag has no effect on the behavior of compaction and we plan to delete + // it in the future. // Specify the file access pattern once a compaction is started. // It will be applied to all input files of a compaction. // Default: NORMAL @@ -923,10 +963,10 @@ struct DBOptions { // running RocksDB on spinning disks, you should set this to at least 2MB. // That way RocksDB's compaction is doing sequential instead of random reads. // - // Default: 0 + // Default: 2MB // // Dynamically changeable through SetDBOptions() API. - size_t compaction_readahead_size = 0; + size_t compaction_readahead_size = 2 * 1024 * 1024; // This is a maximum buffer size that is used by WinMmapReadableFile in // unbuffered disk I/O mode. We need to maintain an aligned buffer for @@ -1166,11 +1206,11 @@ struct DBOptions { // currently. WalFilter* wal_filter = nullptr; - // If true, then DB::Open / CreateColumnFamily / DropColumnFamily + // If true, then DB::Open, CreateColumnFamily, DropColumnFamily, and // SetOptions will fail if options file is not properly persisted. // - // DEFAULT: false - bool fail_if_options_file_error = false; + // DEFAULT: true + bool fail_if_options_file_error = true; // If true, then print malloc stats together with rocksdb.stats // when printing to LOG. @@ -1393,6 +1433,24 @@ struct DBOptions { // of the contract leads to undefined behaviors with high possibility of data // inconsistency, e.g. deleted old data become visible again, etc. bool enforce_single_del_contracts = true; + + // EXPERIMENTAL + // Implementing off-peak duration awareness in RocksDB. In this context, + // "off-peak time" signifies periods characterized by significantly less read + // and write activity compared to other times. By leveraging this knowledge, + // we can prevent low-priority tasks, such as TTL-based compactions, from + // competing with read and write operations during peak hours. Essentially, we + // preprocess these tasks during the preceding off-peak period, just before + // the next peak cycle begins. For example, if the TTL is configured for 25 + // days, we may compact the files during the off-peak hours of the 24th day. + // + // Time of the day in UTC, start_time-end_time inclusive. + // Format - HH:mm-HH:mm (00:00-23:59) + // If the start time > end time, it will be considered that the time period + // spans to the next day (e.g., 23:30-04:00). To make an entire day off-peak, + // use "0:00-23:59". To make an entire day have no offpeak period, leave + // this field blank. Default: Empty string (no offpeak). + std::string daily_offpeak_time_utc = ""; }; // Options to control the behavior of a database (passed to DB::Open) @@ -1519,6 +1577,12 @@ struct ReadOptions { // soft limit then all the remaining keys are returned with status Aborted. uint64_t value_size_soft_limit = std::numeric_limits::max(); + // When the number of merge operands applied exceeds this threshold + // during a successful query, the operation will return a special OK + // Status with subcode kMergeOperandThresholdExceeded. Currently only applies + // to point lookups and is disabled by default. + std::optional merge_operand_count_threshold; + // If true, all data read from underlying storage will be // verified against corresponding checksums. bool verify_checksums = true; @@ -1537,8 +1601,6 @@ struct ReadOptions { // broken, stale keys could be served in read paths. bool ignore_range_deletions = false; - // Experimental - // // If async_io is enabled, RocksDB will prefetch some of data asynchronously. // RocksDB apply it if reads are sequential and its internal automatic // prefetching. @@ -1675,11 +1737,27 @@ struct ReadOptions { // Default: empty (every table will be scanned) std::function table_filter; + // If auto_readahead_size is set to true, it will auto tune the readahead_size + // during scans internally. + // For this feature to enabled, iterate_upper_bound must also be specified. + // + // NOTE: - Recommended for forward Scans only. + // - If there is a backward scans, this option will be + // disabled internally and won't be enabled again if the forward scan + // is issued again. + // + // Default: true + bool auto_readahead_size = true; + // *** END options only relevant to iterators or scans *** - // ** For RocksDB internal use only ** + // *** BEGIN options for RocksDB internal use only *** + + // EXPERIMENTAL Env::IOActivity io_activity = Env::IOActivity::kUnknown; + // *** END options for RocksDB internal use only *** + ReadOptions() {} ReadOptions(bool _verify_checksums, bool _fill_cache); explicit ReadOptions(Env::IOActivity _io_activity); @@ -2088,11 +2166,25 @@ struct WaitForCompactOptions { // called) If true, Status::Aborted will be returned immediately. If false, // ContinueBackgroundWork() must be called to resume the background jobs. // Otherwise, jobs that were queued, but not scheduled yet may never finish - // and WaitForCompact() may wait indefinitely. + // and WaitForCompact() may wait indefinitely (if timeout is set, it will + // expire and return Status::TimedOut). bool abort_on_pause = false; // A boolean to flush all column families before starting to wait. bool flush = false; + + // A boolean to call Close() after waiting is done. By the time Close() is + // called here, there should be no background jobs in progress and no new + // background jobs should be added. DB may not have been closed if Close() + // returned Aborted status due to unreleased snapshots in the system. See + // comments in DB::Close() for details. + bool close_db = false; + + // Timeout in microseconds for waiting for compaction to complete. + // Status::TimedOut will be returned if timeout expires. + // when timeout == 0, WaitForCompact() will wait as long as there's background + // work to finish. + std::chrono::microseconds timeout = std::chrono::microseconds::zero(); }; } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/port_defs.h b/include/rocksdb/port_defs.h index 9771aacb9..68f1d61d2 100644 --- a/include/rocksdb/port_defs.h +++ b/include/rocksdb/port_defs.h @@ -12,6 +12,10 @@ namespace ROCKSDB_NAMESPACE { +namespace port { +class CondVar; +} + enum class CpuPriority { kIdle = 0, kLow = 1, diff --git a/include/rocksdb/rate_limiter.h b/include/rocksdb/rate_limiter.h index 9cad6edf4..3515b1e95 100644 --- a/include/rocksdb/rate_limiter.h +++ b/include/rocksdb/rate_limiter.h @@ -40,6 +40,15 @@ class RateLimiter { // REQUIRED: bytes_per_second > 0 virtual void SetBytesPerSecond(int64_t bytes_per_second) = 0; + // This API allows user to dynamically change the max bytes can be granted in + // a single refill period (i.e, burst) + // + // REQUIRED: single_burst_bytes > 0. Otherwise `Status::InvalidArgument` will + // be returned. + virtual Status SetSingleBurstBytes(int64_t /* single_burst_bytes */) { + return Status::NotSupported(); + } + // Deprecated. New RateLimiter derived classes should override // Request(const int64_t, const Env::IOPriority, Statistics*) or // Request(const int64_t, const Env::IOPriority, Statistics*, OpType) diff --git a/include/rocksdb/rocksdb_namespace.h b/include/rocksdb/rocksdb_namespace.h index a339ec2aa..856300003 100644 --- a/include/rocksdb/rocksdb_namespace.h +++ b/include/rocksdb/rocksdb_namespace.h @@ -12,5 +12,5 @@ // Normal logic #ifndef ROCKSDB_NAMESPACE -#define ROCKSDB_NAMESPACE rocksdb +#define ROCKSDB_NAMESPACE forstdb #endif diff --git a/include/rocksdb/secondary_cache.h b/include/rocksdb/secondary_cache.h index 7c8828168..b0419b121 100644 --- a/include/rocksdb/secondary_cache.h +++ b/include/rocksdb/secondary_cache.h @@ -11,6 +11,7 @@ #include "rocksdb/advanced_cache.h" #include "rocksdb/customizable.h" +#include "rocksdb/options.h" #include "rocksdb/slice.h" #include "rocksdb/statistics.h" #include "rocksdb/status.h" @@ -79,18 +80,23 @@ class SecondaryCache : public Customizable { // and writes it to this cache tier. OK may be returned even if the insertion // is not made. virtual Status Insert(const Slice& key, Cache::ObjectPtr obj, - const Cache::CacheItemHelper* helper) = 0; + const Cache::CacheItemHelper* helper, + bool force_insert) = 0; // Insert a value from its saved/persistable data (typically uncompressed - // block), as if generated by SaveToCallback/SizeCallback. This can be used - // in "warming up" the cache from some auxiliary source, and like Insert() - // may or may not write it to cache depending on the admission control - // policy, even if the return status is success. + // block), as if generated by SaveToCallback/SizeCallback. The data can be + // compressed, in which case the type argument should specify the + // compression algorithm used. Additionally, the source argument should + // be set to the appropriate tier that will be responsible for + // uncompressing the data. // - // The default implementation only assumes the entry helper's create_cb is - // called at Lookup() time and not Insert() time, so should work for all - // foreseeable implementations. - virtual Status InsertSaved(const Slice& key, const Slice& saved); + // This method can be used in "warming up" the cache from some auxiliary + // source, and like Insert() may or may not write it to cache depending on + // the admission control policy, even if the return status is success. + virtual Status InsertSaved( + const Slice& key, const Slice& saved, + CompressionType type = CompressionType::kNoCompression, + CacheTier source = CacheTier::kVolatileTier) = 0; // Lookup the data for the given key in this cache. The create_cb // will be used to create the object. The handle returned may not be @@ -108,7 +114,7 @@ class SecondaryCache : public Customizable { virtual std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, - bool& kept_in_sec_cache) = 0; + Statistics* stats, bool& kept_in_sec_cache) = 0; // Indicate whether a handle can be erased in this secondary cache. [[nodiscard]] virtual bool SupportForceErase() const = 0; @@ -147,4 +153,70 @@ class SecondaryCache : public Customizable { virtual Status Inflate(size_t /*increase*/) { return Status::NotSupported(); } }; +// A wrapper around a SecondaryCache object. A derived class may selectively +// override methods to implement a different behavior. +class SecondaryCacheWrapper : public SecondaryCache { + public: + explicit SecondaryCacheWrapper(std::shared_ptr target) + : target_(std::move(target)) {} + + virtual Status Insert(const Slice& key, Cache::ObjectPtr obj, + const Cache::CacheItemHelper* helper, + bool force_insert) override { + return target()->Insert(key, obj, helper, force_insert); + } + + virtual Status InsertSaved( + const Slice& key, const Slice& saved, + CompressionType type = CompressionType::kNoCompression, + CacheTier source = CacheTier::kVolatileTier) override { + return target()->InsertSaved(key, saved, type, source); + } + + virtual std::unique_ptr Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool wait, bool advise_erase, + Statistics* stats, bool& kept_in_sec_cache) override { + return target()->Lookup(key, helper, create_context, wait, advise_erase, + stats, kept_in_sec_cache); + } + + virtual bool SupportForceErase() const override { + return target()->SupportForceErase(); + } + + virtual void Erase(const Slice& key) override { target()->Erase(key); } + + virtual void WaitAll( + std::vector handles) override { + target()->WaitAll(handles); + } + + virtual Status SetCapacity(size_t capacity) override { + return target()->SetCapacity(capacity); + } + + virtual Status GetCapacity(size_t& capacity) override { + return target()->GetCapacity(capacity); + } + + virtual Status Deflate(size_t decrease) override { + return target()->Deflate(decrease); + } + + virtual Status Inflate(size_t increase) override { + return target()->Inflate(increase); + } + + protected: + SecondaryCache* target() const { return target_.get(); } + + private: + std::shared_ptr target_; +}; + +// Useful for cache entries that just need to be copied into a +// secondary cache, such as compressed blocks +extern const Cache::CacheItemHelper kSliceCacheItemHelper; + } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/sst_file_writer.h b/include/rocksdb/sst_file_writer.h index fb2806865..a2d52733d 100644 --- a/include/rocksdb/sst_file_writer.h +++ b/include/rocksdb/sst_file_writer.h @@ -13,6 +13,7 @@ #include "rocksdb/options.h" #include "rocksdb/table_properties.h" #include "rocksdb/types.h" +#include "rocksdb/wide_columns.h" #if defined(__GNUC__) || defined(__clang__) #define ROCKSDB_DEPRECATED_FUNC __attribute__((__deprecated__)) @@ -45,7 +46,7 @@ struct ExternalSstFileInfo { const std::string& _smallest_key, const std::string& _largest_key, SequenceNumber _sequence_number, uint64_t _file_size, - int32_t _num_entries, int32_t _version) + uint64_t _num_entries, int32_t _version) : file_path(_file_path), smallest_key(_smallest_key), largest_key(_largest_key), @@ -127,6 +128,10 @@ class SstFileWriter { // REQUIRES: timestamp's size is equal to what is expected by the comparator. Status Put(const Slice& user_key, const Slice& timestamp, const Slice& value); + // Add a PutEntity (key with the wide-column entity defined by "columns") to + // the currently opened file + Status PutEntity(const Slice& user_key, const WideColumns& columns); + // Add a Merge key with value to currently opened file // REQUIRES: user_key is after any previously added point (Put/Merge/Delete) // key according to the comparator. @@ -186,4 +191,3 @@ class SstFileWriter { std::unique_ptr rep_; }; } // namespace ROCKSDB_NAMESPACE - diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index a56b35dbe..9aab33712 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -208,8 +208,11 @@ enum Tickers : uint32_t { // DEPRECATED / unused (see NUMBER_BLOCK_COMPRESSION_*) NUMBER_BLOCK_NOT_COMPRESSED, + + // Tickers that record cumulative time. MERGE_OPERATION_TOTAL_TIME, FILTER_OPERATION_TOTAL_TIME, + COMPACTION_CPU_TOTAL_TIME, // Row cache. ROW_CACHE_HIT, @@ -511,6 +514,29 @@ enum Tickers : uint32_t { // compressed SST blocks from storage. BYTES_DECOMPRESSED_TO, + // Number of times readahead is trimmed during scans when + // ReadOptions.auto_readahead_size is set. + READAHEAD_TRIMMED, + + // Number of FIFO compactions that drop files based on different reasons + FIFO_MAX_SIZE_COMPACTIONS, + FIFO_TTL_COMPACTIONS, + + // Number of bytes prefetched during user initiated scan + PREFETCH_BYTES, + + // Number of prefetched bytes that were actually useful + PREFETCH_BYTES_USEFUL, + + // Number of FS reads avoided due to scan prefetching + PREFETCH_HITS, + + // Compressed secondary cache related stats + COMPRESSED_SECONDARY_CACHE_DUMMY_HITS, + COMPRESSED_SECONDARY_CACHE_HITS, + COMPRESSED_SECONDARY_CACHE_PROMOTIONS, + COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS, + TICKER_ENUM_MAX }; @@ -555,6 +581,13 @@ enum Histograms : uint32_t { FILE_READ_FLUSH_MICROS, FILE_READ_COMPACTION_MICROS, FILE_READ_DB_OPEN_MICROS, + // The following `FILE_READ_*` require stats level greater than + // `StatsLevel::kExceptDetailedTimers` + FILE_READ_GET_MICROS, + FILE_READ_MULTIGET_MICROS, + FILE_READ_DB_ITERATOR_MICROS, + FILE_READ_VERIFY_DB_CHECKSUM_MICROS, + FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS, // The number of subcompactions actually scheduled during a compaction NUM_SUBCOMPACTIONS_SCHEDULED, diff --git a/include/rocksdb/status.h b/include/rocksdb/status.h index 447c3b9fe..82597239f 100644 --- a/include/rocksdb/status.h +++ b/include/rocksdb/status.h @@ -114,6 +114,7 @@ class Status { kTxnNotPrepared = 13, kIOFenced = 14, kMergeOperatorFailed = 15, + kMergeOperandThresholdExceeded = 16, kMaxSubCode }; @@ -150,6 +151,25 @@ class Status { return state_.get(); } + // Override this status with another, unless this status is already non-ok. + // Returns *this. Thus, the result of `a.UpdateIfOk(b).UpdateIfOk(c)` is + // non-ok (and `a` modified as such) iff any input was non-ok, with + // left-most taking precedence as far as the details. + Status& UpdateIfOk(Status&& s) { + if (code() == kOk) { + *this = std::move(s); + } else { + // Alright to ignore that status as long as this one is checked + s.PermitUncheckedError(); + } + MustCheck(); + return *this; + } + + Status& UpdateIfOk(const Status& s) { + return UpdateIfOk(std::forward(Status(s))); + } + // Return a success status. static Status OK() { return Status(); } @@ -159,6 +179,14 @@ class Status { // changing public APIs. static Status OkOverwritten() { return Status(kOk, kOverwritten); } + // Successful, though the number of operands merged during the query exceeded + // the threshold. Note: using variants of OK status for program logic is + // discouraged, but it can be useful for communicating statistical information + // without changing public APIs. + static Status OkMergeOperandThresholdExceeded() { + return Status(kOk, kMergeOperandThresholdExceeded); + } + // Return error status of an appropriate type. static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { return Status(kNotFound, msg, msg2); @@ -301,6 +329,13 @@ class Status { return code() == kOk && subcode() == kOverwritten; } + // Returns true iff the status indicates success *with* the number of operands + // merged exceeding the threshold + bool IsOkMergeOperandThresholdExceeded() const { + MarkChecked(); + return code() == kOk && subcode() == kMergeOperandThresholdExceeded; + } + // Returns true iff the status indicates a NotFound error. bool IsNotFound() const { MarkChecked(); diff --git a/include/rocksdb/system_clock.h b/include/rocksdb/system_clock.h index 7ca92e54e..c4cfcecb5 100644 --- a/include/rocksdb/system_clock.h +++ b/include/rocksdb/system_clock.h @@ -9,9 +9,11 @@ #pragma once #include +#include #include #include "rocksdb/customizable.h" +#include "rocksdb/port_defs.h" #include "rocksdb/rocksdb_namespace.h" #include "rocksdb/status.h" @@ -68,6 +70,14 @@ class SystemClock : public Customizable { // Sleep/delay the thread for the prescribed number of micro-seconds. virtual void SleepForMicroseconds(int micros) = 0; + // For internal use/extension only. + // + // Issues a wait on `cv` that times out at `deadline`. May wakeup and return + // spuriously. + // + // Returns true if wait timed out, false otherwise + virtual bool TimedWait(port::CondVar* cv, std::chrono::microseconds deadline); + // Get the number of seconds since the Epoch, 1970-01-01 00:00:00 (UTC). // Only overwrites *unix_time on success. virtual Status GetCurrentTime(int64_t* unix_time) = 0; @@ -94,6 +104,11 @@ class SystemClockWrapper : public SystemClock { return target_->SleepForMicroseconds(micros); } + virtual bool TimedWait(port::CondVar* cv, + std::chrono::microseconds deadline) override { + return target_->TimedWait(cv, deadline); + } + Status GetCurrentTime(int64_t* unix_time) override { return target_->GetCurrentTime(unix_time); } diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h index 6e8f60577..d19a95fa8 100644 --- a/include/rocksdb/table.h +++ b/include/rocksdb/table.h @@ -47,7 +47,10 @@ struct EnvOptions; // Types of checksums to use for checking integrity of logical blocks within // files. All checksums currently use 32 bits of checking power (1 in 4B -// chance of failing to detect random corruption). +// chance of failing to detect random corruption). Traditionally, the actual +// checking power can be far from ideal if the corruption is due to misplaced +// data (e.g. physical blocks out of order in a file, or from another file), +// which is fixed in format_version=6 (see below). enum ChecksumType : char { kNoChecksum = 0x0, kCRC32c = 0x1, @@ -512,6 +515,9 @@ struct BlockBasedTableOptions { // 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned // filters use a generally faster and more accurate Bloom filter // implementation, with a different schema. + // 6 -- Modified the file footer and checksum matching so that SST data + // misplaced within or between files is as likely to fail checksum + // verification as random corruption. Also checksum-protects SST footer. uint32_t format_version = 5; // Store index blocks on disk in compressed format. Changing this option to diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h index ebde339dd..b87647a0d 100644 --- a/include/rocksdb/table_properties.h +++ b/include/rocksdb/table_properties.h @@ -122,12 +122,15 @@ class TablePropertiesCollector { // Finish() will be called when a table has already been built and is ready // for writing the properties block. + // It will be called only once by RocksDB internal. + // // @params properties User will add their collected statistics to // `properties`. virtual Status Finish(UserCollectedProperties* properties) = 0; // Return the human-readable properties, where the key is property name and // the value is the human-readable form of value. + // It will only be called after Finish() has been called by RocksDB internal. virtual UserCollectedProperties GetReadableProperties() const = 0; // The name of the properties collector can be used for debugging purpose. @@ -137,8 +140,7 @@ class TablePropertiesCollector { virtual bool NeedCompact() const { return false; } }; -// Constructs TablePropertiesCollector. Internals create a new -// TablePropertiesCollector for each new table +// Constructs TablePropertiesCollector instances for each table file creation. // // Exceptions MUST NOT propagate out of overridden functions into RocksDB, // because RocksDB is not exception-safe. This could cause undefined behavior @@ -160,7 +162,12 @@ class TablePropertiesCollectorFactory : public Customizable { const ConfigOptions& options, const std::string& value, std::shared_ptr* result); - // has to be thread-safe + // To collect properties of a table with the given context, returns + // a new object inheriting from TablePropertiesCollector. The caller + // is responsible for deleting the object returned. Alternatively, + // nullptr may be returned to decline collecting properties for the + // file (and reduce callback overheads). + // MUST be thread-safe. virtual TablePropertiesCollector* CreateTablePropertiesCollector( TablePropertiesCollectorFactory::Context context) = 0; @@ -219,9 +226,20 @@ struct TableProperties { // by column_family_name. uint64_t column_family_id = ROCKSDB_NAMESPACE:: TablePropertiesCollectorFactory::Context::kUnknownColumnFamily; - // Timestamp of the latest key. 0 means unknown. - // TODO(sagar0): Should be changed to latest_key_time ... but don't know the - // full implications of backward compatibility. Hence retaining for now. + + // Oldest ancester time. 0 means unknown. + // + // For flush output file, oldest ancestor time is the oldest key time in the + // file. If the oldest key time is not available, flush time is used. + // + // For compaction output file, oldest ancestor time is the oldest + // among all the oldest key time of its input files, since the file could be + // the compaction output from other SST files, which could in turn be outputs + // for compact older SST files. If that's not available, creation time of this + // compaction output file is used. + // + // TODO(sagar0): Should be changed to oldest_ancester_time ... but don't know + // the full implications of backward compatibility. Hence retaining for now. uint64_t creation_time = 0; // Timestamp of the earliest key. 0 means unknown. diff --git a/include/rocksdb/thread_status.h b/include/rocksdb/thread_status.h index beecdfd25..5bc6eeb2b 100644 --- a/include/rocksdb/thread_status.h +++ b/include/rocksdb/thread_status.h @@ -57,6 +57,13 @@ struct ThreadStatus { OP_COMPACTION, OP_FLUSH, OP_DBOPEN, + OP_GET, + OP_MULTIGET, + OP_DBITERATOR, + OP_VERIFY_DB_CHECKSUM, + OP_VERIFY_FILE_CHECKSUMS, + OP_GETENTITY, + OP_MULTIGETENTITY, NUM_OP_TYPES }; diff --git a/include/rocksdb/types.h b/include/rocksdb/types.h index 3f8ce9795..c9c214686 100644 --- a/include/rocksdb/types.h +++ b/include/rocksdb/types.h @@ -7,6 +7,9 @@ #include +#include +#include + #include "rocksdb/slice.h" namespace ROCKSDB_NAMESPACE { @@ -18,6 +21,10 @@ using ColumnFamilyId = uint32_t; // Represents a sequence number in a WAL file. using SequenceNumber = uint64_t; +struct TableProperties; +using TablePropertiesCollection = + std::unordered_map>; + const SequenceNumber kMinUnCommittedSeq = 1; // 0 is always committed enum class TableFileCreationReason { diff --git a/include/rocksdb/utilities/ldb_cmd.h b/include/rocksdb/utilities/ldb_cmd.h index af5ee4ba9..ed4f5de7e 100644 --- a/include/rocksdb/utilities/ldb_cmd.h +++ b/include/rocksdb/utilities/ldb_cmd.h @@ -226,6 +226,12 @@ class LDBCommand { static std::string PrintKeyValue(const std::string& key, const std::string& value, bool is_hex); + static std::string PrintKeyValueOrWideColumns(const Slice& key, + const Slice& value, + const WideColumns& wide_columns, + bool is_key_hex, + bool is_value_hex); + /** * Return true if the specified flag is present in the specified flags vector */ @@ -313,4 +319,3 @@ class LDBCommandRunner { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/include/rocksdb/utilities/options_type.h b/include/rocksdb/utilities/options_type.h index cd340ed59..782b14e65 100644 --- a/include/rocksdb/utilities/options_type.h +++ b/include/rocksdb/utilities/options_type.h @@ -40,8 +40,9 @@ enum class OptionType { kUInt32T, kUInt64T, kSizeT, - kString, kDouble, + kAtomicInt, + kString, kCompactionStyle, kCompactionPri, kCompressionType, diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index 8674f10c9..86e1477a4 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -92,6 +92,10 @@ class StackableDB : public DB { const WideColumns& columns) override { return db_->PutEntity(options, column_family, key, columns); } + Status PutEntity(const WriteOptions& options, const Slice& key, + const AttributeGroups& attribute_groups) override { + return db_->PutEntity(options, key, attribute_groups); + } using DB::Get; virtual Status Get(const ReadOptions& options, @@ -593,6 +597,8 @@ class StackableDB : public DB { return db_->TryCatchUpWithPrimary(); } + virtual Status Resume() override { return db_->Resume(); } + protected: DB* db_; std::shared_ptr shared_db_ptr_; diff --git a/include/rocksdb/utilities/transaction.h b/include/rocksdb/utilities/transaction.h index 947fcec55..e6452056a 100644 --- a/include/rocksdb/utilities/transaction.h +++ b/include/rocksdb/utilities/transaction.h @@ -227,7 +227,8 @@ class Transaction { // Status::Busy() may be returned if the transaction could not guarantee // that there are no write conflicts. Status::TryAgain() may be returned // if the memtable history size is not large enough - // (See max_write_buffer_size_to_maintain). + // (see max_write_buffer_size_to_maintain). In either case, a Rollback() + // or new transaction is required to expect a different result. // // If this transaction was created by a TransactionDB(), Status::Expired() // may be returned if this transaction has lived for longer than @@ -259,6 +260,7 @@ class Transaction { std::shared_ptr* snapshot = nullptr); // Discard all batched writes in this transaction. + // FIXME: what happens if this isn't called before destruction? virtual Status Rollback() = 0; // Records the state of the transaction for future calls to @@ -333,8 +335,22 @@ class Transaction { const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, const bool /*sorted_input*/ = false) { + if (options.io_activity != Env::IOActivity::kUnknown && + options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } + for (size_t i = 0; i < num_keys; ++i) { - statuses[i] = Get(options, column_family, keys[i], &values[i]); + statuses[i] = GetImpl(options, column_family, keys[i], &values[i]); } } @@ -511,6 +527,15 @@ class Transaction { virtual Status SingleDeleteUntracked(const Slice& key) = 0; + // Collpase the merge chain for the given key. This is can be used by the + // application to trigger an on-demand collpase to a key that has a long + // merge chain to reduce read amplification, without waiting for compaction + // to kick in. + virtual Status CollapseKey(const ReadOptions&, const Slice&, + ColumnFamilyHandle* = nullptr) { + return Status::NotSupported("collpase not supported"); + } + // Similar to WriteBatch::PutLogData virtual void PutLogData(const Slice& blob) = 0; @@ -671,6 +696,21 @@ class Transaction { id_ = id; } + virtual Status GetImpl(const ReadOptions& /* options */, + ColumnFamilyHandle* /* column_family */, + const Slice& /* key */, std::string* /* value */) { + return Status::NotSupported("Not implemented"); + } + + virtual Status GetImpl(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* pinnable_val) { + assert(pinnable_val != nullptr); + auto s = GetImpl(options, column_family, key, pinnable_val->GetSelf()); + pinnable_val->PinSelf(); + return s; + } + virtual uint64_t GetLastLogNumber() const { return log_number_; } private: diff --git a/include/rocksdb/utilities/write_batch_with_index.h b/include/rocksdb/utilities/write_batch_with_index.h index d5867567b..090a4a444 100644 --- a/include/rocksdb/utilities/write_batch_with_index.h +++ b/include/rocksdb/utilities/write_batch_with_index.h @@ -10,7 +10,6 @@ // inserted. #pragma once - #include #include #include @@ -30,6 +29,7 @@ class DB; class ReadCallback; struct ReadOptions; struct DBOptions; +class MergeContext; enum WriteType { kPutRecord, @@ -39,11 +39,12 @@ enum WriteType { kDeleteRangeRecord, kLogDataRecord, kXIDRecord, + kPutEntityRecord, kUnknownRecord, }; -// an entry for Put, Merge, Delete, or SingleDelete entry for write batches. -// Used in WBWIIterator. +// An entry for Put, PutEntity, Merge, Delete, or SingleDelete for write +// batches. Used in WBWIIterator. struct WriteEntry { WriteType type = kUnknownRecord; Slice key; @@ -77,12 +78,11 @@ class WBWIIterator { }; // A WriteBatchWithIndex with a binary searchable index built for all the keys -// inserted. -// In Put(), Merge() Delete(), or SingleDelete(), the same function of the -// wrapped will be called. At the same time, indexes will be built. -// By calling GetWriteBatch(), a user will get the WriteBatch for the data -// they inserted, which can be used for DB::Write(). -// A user can call NewIterator() to create an iterator. +// inserted. In Put(), PutEntity(), Merge(), Delete(), or SingleDelete(), the +// corresponding function of the wrapped WriteBatch will be called. At the same +// time, indexes will be built. By calling GetWriteBatch(), a user will get the +// WriteBatch for the data they inserted, which can be used for DB::Write(). A +// user can call NewIterator() to create an iterator. class WriteBatchWithIndex : public WriteBatchBase { public: // backup_index_comparator: the backup comparator used to compare keys @@ -112,13 +112,23 @@ class WriteBatchWithIndex : public WriteBatchBase { Status Put(ColumnFamilyHandle* column_family, const Slice& key, const Slice& ts, const Slice& value) override; + using WriteBatchBase::TimedPut; + Status TimedPut(ColumnFamilyHandle* /* column_family */, + const Slice& /* key */, const Slice& /* value */, + uint64_t /* write_unix_time */) override { + return Status::NotSupported( + "TimedPut not supported by WriteBatchWithIndex"); + } + Status PutEntity(ColumnFamilyHandle* column_family, const Slice& /* key */, - const WideColumns& /* columns */) override { - if (!column_family) { + const WideColumns& /* columns */) override; + + Status PutEntity(const Slice& /* key */, + const AttributeGroups& attribute_groups) override { + if (attribute_groups.empty()) { return Status::InvalidArgument( - "Cannot call this method without a column family handle"); + "Cannot call this method without attribute groups"); } - return Status::NotSupported( "PutEntity not supported by WriteBatchWithIndex"); } @@ -189,9 +199,6 @@ class WriteBatchWithIndex : public WriteBatchBase { // Will create a new Iterator that will use WBWIIterator as a delta and // base_iterator as base. // - // This function is only supported if the WriteBatchWithIndex was - // constructed with overwrite_key=true. - // // The returned iterator should be deleted by the caller. // The base_iterator is now 'owned' by the returned iterator. Deleting the // returned iterator will also delete the base_iterator. @@ -222,6 +229,8 @@ class WriteBatchWithIndex : public WriteBatchBase { return GetFromBatch(nullptr, options, key, value); } + // TODO: implement GetEntityFromBatch + // Similar to DB::Get() but will also read writes from this batch. // // This function will query both this batch and the DB and then merge @@ -248,21 +257,24 @@ class WriteBatchWithIndex : public WriteBatchBase { ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value); + // TODO: implement GetEntityFromBatchAndDB + void MultiGetFromBatchAndDB(DB* db, const ReadOptions& read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, bool sorted_input); + // TODO: implement MultiGetEntityFromBatchAndDB + // Records the state of the batch for future calls to RollbackToSavePoint(). // May be called multiple times to set multiple save points. void SetSavePoint() override; - // Remove all entries in this batch (Put, Merge, Delete, SingleDelete, - // PutLogData) since the most recent call to SetSavePoint() and removes the - // most recent save point. - // If there is no previous call to SetSavePoint(), behaves the same as - // Clear(). + // Remove all entries in this batch (Put, PutEntity, Merge, Delete, + // SingleDelete, PutLogData) since the most recent call to SetSavePoint() and + // removes the most recent save point. If there is no previous call to + // SetSavePoint(), behaves the same as Clear(). // // Calling RollbackToSavePoint invalidates any open iterators on this batch. // @@ -291,6 +303,11 @@ class WriteBatchWithIndex : public WriteBatchBase { // last sub-batch. size_t SubBatchCnt(); + void MergeAcrossBatchAndDB(ColumnFamilyHandle* column_family, + const Slice& key, + const PinnableWideColumns& existing, + const MergeContext& merge_context, + PinnableSlice* value, Status* status); Status GetFromBatchAndDB(DB* db, const ReadOptions& read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value, ReadCallback* callback); @@ -304,4 +321,3 @@ class WriteBatchWithIndex : public WriteBatchBase { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/include/rocksdb/version.h b/include/rocksdb/version.h index a19f41fd2..530d4fee7 100644 --- a/include/rocksdb/version.h +++ b/include/rocksdb/version.h @@ -12,8 +12,8 @@ // NOTE: in 'main' development branch, this should be the *next* // minor or major version number planned for release. #define ROCKSDB_MAJOR 8 -#define ROCKSDB_MINOR 5 -#define ROCKSDB_PATCH 3 +#define ROCKSDB_MINOR 10 +#define ROCKSDB_PATCH 0 // Do not use these. We made the mistake of declaring macros starting with // double underscore. Now we have to live with our choice. We'll deprecate these diff --git a/include/rocksdb/wide_columns.h b/include/rocksdb/wide_columns.h index 5af3f51de..35b81268b 100644 --- a/include/rocksdb/wide_columns.h +++ b/include/rocksdb/wide_columns.h @@ -16,6 +16,8 @@ namespace ROCKSDB_NAMESPACE { +class ColumnFamilyHandle; + // Class representing a wide column, which is defined as a pair of column name // and column value. class WideColumn { @@ -74,8 +76,19 @@ inline bool operator!=(const WideColumn& lhs, const WideColumn& rhs) { inline std::ostream& operator<<(std::ostream& os, const WideColumn& column) { const bool hex = (os.flags() & std::ios_base::basefield) == std::ios_base::hex; - os << column.name().ToString(hex) << ':' << column.value().ToString(hex); - + if (!column.name().empty()) { + if (hex) { + os << "0x"; + } + os << column.name().ToString(hex); + } + os << ':'; + if (!column.value().empty()) { + if (hex) { + os << "0x"; + } + os << column.value().ToString(hex); + } return os; } @@ -207,4 +220,61 @@ inline bool operator!=(const PinnableWideColumns& lhs, return !(lhs == rhs); } +// Class representing attribute group. Attribute group is a logical grouping of +// wide-column entities by leveraging Column Families. +// Used in Write Path +class AttributeGroup { + public: + ColumnFamilyHandle* column_family() const { return column_family_; } + const WideColumns& columns() const { return columns_; } + WideColumns& columns() { return columns_; } + + explicit AttributeGroup(ColumnFamilyHandle* column_family, + const WideColumns& columns) + : column_family_(column_family), columns_(columns) {} + + private: + ColumnFamilyHandle* column_family_; + WideColumns columns_; +}; + +// A collection of Attribute Groups. +using AttributeGroups = std::vector; + +// Used in Read Path. Wide-columns returned from the query are pinnable. +class PinnableAttributeGroup { + public: + ColumnFamilyHandle* column_family() const { return column_family_; } + const Status& status() const { return status_; } + const WideColumns& columns() const { return columns_.columns(); } + + explicit PinnableAttributeGroup(ColumnFamilyHandle* column_family) + : column_family_(column_family), status_(Status::OK()) {} + + void SetStatus(const Status& status); + void SetColumns(PinnableWideColumns&& columns); + + void Reset(); + + private: + ColumnFamilyHandle* column_family_; + Status status_; + PinnableWideColumns columns_; +}; + +inline void PinnableAttributeGroup::SetStatus(const Status& status) { + status_ = status; +} +inline void PinnableAttributeGroup::SetColumns(PinnableWideColumns&& columns) { + columns_ = std::move(columns); +} + +inline void PinnableAttributeGroup::Reset() { + SetStatus(Status::OK()); + columns_.Reset(); +} + +// A collection of Pinnable Attribute Groups. +using PinnableAttributeGroups = std::vector; + } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/write_batch.h b/include/rocksdb/write_batch.h index 307cd7559..5c87f9405 100644 --- a/include/rocksdb/write_batch.h +++ b/include/rocksdb/write_batch.h @@ -43,13 +43,13 @@ struct SavePoints; struct SliceParts; struct SavePoint { - size_t size; // size of rep_ - int count; // count of elements in rep_ + size_t size; // size of rep_ + uint32_t count; // count of elements in rep_ uint32_t content_flags; SavePoint() : size(0), count(0), content_flags(0) {} - SavePoint(size_t _size, int _count, uint32_t _flags) + SavePoint(size_t _size, uint32_t _count, uint32_t _flags) : size(_size), count(_count), content_flags(_flags) {} void clear() { @@ -100,12 +100,28 @@ class WriteBatch : public WriteBatchBase { return Put(nullptr, key, value); } + using WriteBatchBase::TimedPut; + // DO NOT USE, UNDER CONSTRUCTION + // Stores the mapping "key->value" in the database with the specified write + // time in the column family. + Status TimedPut(ColumnFamilyHandle* /* column_family */, + const Slice& /* key */, const Slice& /* value */, + uint64_t /* write_unix_time */) override { + // TODO(yuzhangyu): implement take in the write time. + return Status::NotSupported("TimedPut is under construction"); + } + // Store the mapping "key->{column1:value1, column2:value2, ...}" in the // column family specified by "column_family". using WriteBatchBase::PutEntity; Status PutEntity(ColumnFamilyHandle* column_family, const Slice& key, const WideColumns& columns) override; + // Split and store wide column entities in multiple column families (a.k.a. + // AttributeGroups) + Status PutEntity(const Slice& key, + const AttributeGroups& attribute_groups) override; + using WriteBatchBase::Delete; // If the database contains a mapping for "key", erase it. Else do nothing. // The following Delete(..., const Slice& key) can be used when user-defined @@ -395,8 +411,6 @@ class WriteBatch : public WriteBatchBase { // Returns true if MarkRollback will be called during Iterate bool HasRollback() const; - // Experimental. - // // Update timestamps of existing entries in the write batch if // applicable. If a key is intended for a column family that disables // timestamp, then this API won't set the timestamp for this key. diff --git a/include/rocksdb/write_batch_base.h b/include/rocksdb/write_batch_base.h index f6f39ef0b..5b26ee543 100644 --- a/include/rocksdb/write_batch_base.h +++ b/include/rocksdb/write_batch_base.h @@ -42,11 +42,32 @@ class WriteBatchBase { const SliceParts& value); virtual Status Put(const SliceParts& key, const SliceParts& value); + // Store the mapping "key->value" in the database with the specified write + // time in the column family. Using some write time that is in the past to + // fast track data to their correct placement and preservation is the intended + // usage of this API. The DB makes a reasonable best effort to treat the data + // as having the given write time for this purpose but doesn't currently make + // any guarantees. + // + // When a regular Put("foo", "v1") is followed by a + // TimedPut("foo", "v2", some_time_before_first_put), the behavior of read + // queries are undefined and can change over time, for example due to + // compactions. + // Note: this feature is currently not compatible with user-defined timestamps + // and wide columns. + virtual Status TimedPut(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value, uint64_t write_unix_time) = 0; + // Store the mapping "key->{column1:value1, column2:value2, ...}" in the // column family specified by "column_family". virtual Status PutEntity(ColumnFamilyHandle* column_family, const Slice& key, const WideColumns& columns) = 0; + // Split and store wide column entities in multiple column families (a.k.a. + // AttributeGroups) + virtual Status PutEntity(const Slice& key, + const AttributeGroups& attribute_groups) = 0; + // Merge "value" with the existing value of "key" in the database. // "key->merge(existing, value)" virtual Status Merge(ColumnFamilyHandle* column_family, const Slice& key, diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 076800414..4c9a8ff8a 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -1,5 +1,12 @@ cmake_minimum_required(VERSION 3.4) +set(JAVA_JUNIT_VERSION "4.13.1") +set(JAVA_HAMCR_VERSION "2.2") +set(JAVA_MOCKITO_VERSION "1.10.19") +set(JAVA_CGLIB_VERSION "3.3.0") +set(JAVA_ASSERTJ_VERSION "2.9.0") + + if(${CMAKE_VERSION} VERSION_LESS "3.11.4") message("Please consider switching to CMake 3.11.4 or newer") endif() @@ -7,298 +14,507 @@ endif() set(CMAKE_JAVA_COMPILE_FLAGS -source 8) set(JNI_NATIVE_SOURCES - rocksjni/backup_engine_options.cc - rocksjni/backupenginejni.cc - rocksjni/cassandra_compactionfilterjni.cc - rocksjni/cassandra_value_operator.cc - rocksjni/checkpoint.cc - rocksjni/clock_cache.cc - rocksjni/cache.cc - rocksjni/columnfamilyhandle.cc - rocksjni/compaction_filter.cc - rocksjni/compaction_filter_factory.cc - rocksjni/compaction_filter_factory_jnicallback.cc - rocksjni/compaction_job_info.cc - rocksjni/compaction_job_stats.cc - rocksjni/compaction_options.cc - rocksjni/compaction_options_fifo.cc - rocksjni/compaction_options_universal.cc - rocksjni/compact_range_options.cc - rocksjni/comparator.cc - rocksjni/comparatorjnicallback.cc - rocksjni/compression_options.cc - rocksjni/concurrent_task_limiter.cc - rocksjni/config_options.cc - rocksjni/env.cc - rocksjni/env_flink.cc - rocksjni/env_flink_test_suite.cc - rocksjni/env_options.cc - rocksjni/event_listener.cc - rocksjni/event_listener_jnicallback.cc - rocksjni/flink_compactionfilterjni.cc - rocksjni/export_import_files_metadatajni.cc - rocksjni/filter.cc - rocksjni/import_column_family_options.cc - rocksjni/ingest_external_file_options.cc - rocksjni/iterator.cc - rocksjni/jnicallback.cc - rocksjni/loggerjnicallback.cc - rocksjni/lru_cache.cc - rocksjni/memory_util.cc - rocksjni/memtablejni.cc - rocksjni/merge_operator.cc - rocksjni/native_comparator_wrapper_test.cc - rocksjni/optimistic_transaction_db.cc - rocksjni/optimistic_transaction_options.cc - rocksjni/options.cc - rocksjni/options_util.cc - rocksjni/persistent_cache.cc - rocksjni/ratelimiterjni.cc - rocksjni/remove_emptyvalue_compactionfilterjni.cc - rocksjni/restorejni.cc - rocksjni/rocks_callback_object.cc - rocksjni/rocksdb_exception_test.cc - rocksjni/rocksjni.cc - rocksjni/slice.cc - rocksjni/snapshot.cc - rocksjni/sst_file_manager.cc - rocksjni/sst_file_writerjni.cc - rocksjni/sst_file_readerjni.cc - rocksjni/sst_file_reader_iterator.cc - rocksjni/sst_partitioner.cc - rocksjni/statistics.cc - rocksjni/statisticsjni.cc - rocksjni/table.cc - rocksjni/table_filter.cc - rocksjni/table_filter_jnicallback.cc - rocksjni/testable_event_listener.cc - rocksjni/thread_status.cc - rocksjni/trace_writer.cc - rocksjni/trace_writer_jnicallback.cc - rocksjni/transaction.cc - rocksjni/transaction_db.cc - rocksjni/transaction_db_options.cc - rocksjni/transaction_log.cc - rocksjni/transaction_notifier.cc - rocksjni/transaction_notifier_jnicallback.cc - rocksjni/transaction_options.cc - rocksjni/ttl.cc - rocksjni/wal_filter.cc - rocksjni/wal_filter_jnicallback.cc - rocksjni/write_batch.cc - rocksjni/writebatchhandlerjnicallback.cc - rocksjni/write_batch_test.cc - rocksjni/write_batch_with_index.cc - rocksjni/write_buffer_manager.cc + forstjni/backup_engine_options.cc + forstjni/backupenginejni.cc + forstjni/cassandra_compactionfilterjni.cc + forstjni/cassandra_value_operator.cc + forstjni/checkpoint.cc + forstjni/clock_cache.cc + forstjni/cache.cc + forstjni/columnfamilyhandle.cc + forstjni/compaction_filter.cc + forstjni/compaction_filter_factory.cc + forstjni/compaction_filter_factory_jnicallback.cc + forstjni/compaction_job_info.cc + forstjni/compaction_job_stats.cc + forstjni/compaction_options.cc + forstjni/compaction_options_fifo.cc + forstjni/compaction_options_universal.cc + forstjni/compact_range_options.cc + forstjni/comparator.cc + forstjni/comparatorjnicallback.cc + forstjni/compression_options.cc + forstjni/concurrent_task_limiter.cc + forstjni/config_options.cc + forstjni/env.cc + forstjni/env_flink.cc + forstjni/env_flink_test_suite.cc + forstjni/env_options.cc + forstjni/event_listener.cc + forstjni/event_listener_jnicallback.cc + forstjni/export_import_files_metadatajni.cc + forstjni/flink_compactionfilterjni.cc + forstjni/filter.cc + forstjni/import_column_family_options.cc + forstjni/hyper_clock_cache.cc + forstjni/ingest_external_file_options.cc + forstjni/iterator.cc + forstjni/jnicallback.cc + forstjni/loggerjnicallback.cc + forstjni/lru_cache.cc + forstjni/memory_util.cc + forstjni/memtablejni.cc + forstjni/merge_operator.cc + forstjni/native_comparator_wrapper_test.cc + forstjni/optimistic_transaction_db.cc + forstjni/optimistic_transaction_options.cc + forstjni/options.cc + forstjni/options_util.cc + forstjni/persistent_cache.cc + forstjni/jni_perf_context.cc + forstjni/ratelimiterjni.cc + forstjni/remove_emptyvalue_compactionfilterjni.cc + forstjni/restorejni.cc + forstjni/rocks_callback_object.cc + forstjni/rocksdb_exception_test.cc + forstjni/rocksjni.cc + forstjni/slice.cc + forstjni/snapshot.cc + forstjni/sst_file_manager.cc + forstjni/sst_file_writerjni.cc + forstjni/sst_file_readerjni.cc + forstjni/sst_file_reader_iterator.cc + forstjni/sst_partitioner.cc + forstjni/statistics.cc + forstjni/statisticsjni.cc + forstjni/table.cc + forstjni/table_filter.cc + forstjni/table_filter_jnicallback.cc + forstjni/testable_event_listener.cc + forstjni/thread_status.cc + forstjni/trace_writer.cc + forstjni/trace_writer_jnicallback.cc + forstjni/transaction.cc + forstjni/transaction_db.cc + forstjni/transaction_db_options.cc + forstjni/transaction_log.cc + forstjni/transaction_notifier.cc + forstjni/transaction_notifier_jnicallback.cc + forstjni/transaction_options.cc + forstjni/ttl.cc + forstjni/wal_filter.cc + forstjni/wal_filter_jnicallback.cc + forstjni/write_batch.cc + forstjni/writebatchhandlerjnicallback.cc + forstjni/write_batch_test.cc + forstjni/write_batch_with_index.cc + forstjni/write_buffer_manager.cc ) set(JAVA_MAIN_CLASSES - src/main/java/org/rocksdb/AbstractCompactionFilter.java - src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java - src/main/java/org/rocksdb/AbstractComparator.java - src/main/java/org/rocksdb/AbstractEventListener.java - src/main/java/org/rocksdb/AbstractImmutableNativeReference.java - src/main/java/org/rocksdb/AbstractMutableOptions.java - src/main/java/org/rocksdb/AbstractNativeReference.java - src/main/java/org/rocksdb/AbstractRocksIterator.java - src/main/java/org/rocksdb/AbstractSlice.java - src/main/java/org/rocksdb/AbstractTableFilter.java - src/main/java/org/rocksdb/AbstractTraceWriter.java - src/main/java/org/rocksdb/AbstractTransactionNotifier.java - src/main/java/org/rocksdb/AbstractWalFilter.java - src/main/java/org/rocksdb/AbstractWriteBatch.java - src/main/java/org/rocksdb/AccessHint.java - src/main/java/org/rocksdb/AdvancedColumnFamilyOptionsInterface.java - src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java - src/main/java/org/rocksdb/BackgroundErrorReason.java - src/main/java/org/rocksdb/BackupEngineOptions.java - src/main/java/org/rocksdb/BackupEngine.java - src/main/java/org/rocksdb/BackupInfo.java - src/main/java/org/rocksdb/BlockBasedTableConfig.java - src/main/java/org/rocksdb/BloomFilter.java - src/main/java/org/rocksdb/BuiltinComparator.java - src/main/java/org/rocksdb/ByteBufferGetStatus.java - src/main/java/org/rocksdb/Cache.java - src/main/java/org/rocksdb/CassandraCompactionFilter.java - src/main/java/org/rocksdb/CassandraValueMergeOperator.java - src/main/java/org/rocksdb/Checkpoint.java - src/main/java/org/rocksdb/ChecksumType.java - src/main/java/org/rocksdb/ClockCache.java - src/main/java/org/rocksdb/ColumnFamilyDescriptor.java - src/main/java/org/rocksdb/ColumnFamilyHandle.java - src/main/java/org/rocksdb/ColumnFamilyMetaData.java - src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java - src/main/java/org/rocksdb/ColumnFamilyOptions.java - src/main/java/org/rocksdb/CompactionJobInfo.java - src/main/java/org/rocksdb/CompactionJobStats.java - src/main/java/org/rocksdb/CompactionOptions.java - src/main/java/org/rocksdb/CompactionOptionsFIFO.java - src/main/java/org/rocksdb/CompactionOptionsUniversal.java - src/main/java/org/rocksdb/CompactionPriority.java - src/main/java/org/rocksdb/CompactionReason.java - src/main/java/org/rocksdb/CompactRangeOptions.java - src/main/java/org/rocksdb/CompactionStopStyle.java - src/main/java/org/rocksdb/CompactionStyle.java - src/main/java/org/rocksdb/ComparatorOptions.java - src/main/java/org/rocksdb/ComparatorType.java - src/main/java/org/rocksdb/CompressionOptions.java - src/main/java/org/rocksdb/CompressionType.java - src/main/java/org/rocksdb/ConfigOptions.java - src/main/java/org/rocksdb/DataBlockIndexType.java - src/main/java/org/rocksdb/DBOptionsInterface.java - src/main/java/org/rocksdb/DBOptions.java - src/main/java/org/rocksdb/DbPath.java - src/main/java/org/rocksdb/DirectSlice.java - src/main/java/org/rocksdb/EncodingType.java - src/main/java/org/rocksdb/Env.java - src/main/java/org/rocksdb/EnvFlinkTestSuite.java - src/main/java/org/rocksdb/EnvOptions.java - src/main/java/org/rocksdb/EventListener.java - src/main/java/org/rocksdb/Experimental.java - src/main/java/org/rocksdb/ExportImportFilesMetaData.java - src/main/java/org/rocksdb/ExternalFileIngestionInfo.java - src/main/java/org/rocksdb/Filter.java - src/main/java/org/rocksdb/FileOperationInfo.java - src/main/java/org/rocksdb/FlinkCompactionFilter.java - src/main/java/org/rocksdb/FlinkEnv.java - src/main/java/org/rocksdb/FlushJobInfo.java - src/main/java/org/rocksdb/FlushReason.java - src/main/java/org/rocksdb/FlushOptions.java - src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java - src/main/java/org/rocksdb/HashSkipListMemTableConfig.java - src/main/java/org/rocksdb/HistogramData.java - src/main/java/org/rocksdb/HistogramType.java - src/main/java/org/rocksdb/Holder.java - src/main/java/org/rocksdb/ImportColumnFamilyOptions.java - src/main/java/org/rocksdb/IndexShorteningMode.java - src/main/java/org/rocksdb/IndexType.java - src/main/java/org/rocksdb/InfoLogLevel.java - src/main/java/org/rocksdb/IngestExternalFileOptions.java - src/main/java/org/rocksdb/LevelMetaData.java - src/main/java/org/rocksdb/ConcurrentTaskLimiter.java - src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java - src/main/java/org/rocksdb/KeyMayExist.java - src/main/java/org/rocksdb/LiveFileMetaData.java - src/main/java/org/rocksdb/LogFile.java - src/main/java/org/rocksdb/Logger.java - src/main/java/org/rocksdb/LRUCache.java - src/main/java/org/rocksdb/MemoryUsageType.java - src/main/java/org/rocksdb/MemoryUtil.java - src/main/java/org/rocksdb/MemTableConfig.java - src/main/java/org/rocksdb/MemTableInfo.java - src/main/java/org/rocksdb/MergeOperator.java - src/main/java/org/rocksdb/MutableColumnFamilyOptions.java - src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java - src/main/java/org/rocksdb/MutableDBOptions.java - src/main/java/org/rocksdb/MutableDBOptionsInterface.java - src/main/java/org/rocksdb/MutableOptionKey.java - src/main/java/org/rocksdb/MutableOptionValue.java - src/main/java/org/rocksdb/NativeComparatorWrapper.java - src/main/java/org/rocksdb/NativeLibraryLoader.java - src/main/java/org/rocksdb/OperationStage.java - src/main/java/org/rocksdb/OperationType.java - src/main/java/org/rocksdb/OptimisticTransactionDB.java - src/main/java/org/rocksdb/OptimisticTransactionOptions.java - src/main/java/org/rocksdb/Options.java - src/main/java/org/rocksdb/OptionString.java - src/main/java/org/rocksdb/OptionsUtil.java - src/main/java/org/rocksdb/PersistentCache.java - src/main/java/org/rocksdb/PlainTableConfig.java - src/main/java/org/rocksdb/PrepopulateBlobCache.java - src/main/java/org/rocksdb/Priority.java - src/main/java/org/rocksdb/Range.java - src/main/java/org/rocksdb/RateLimiter.java - src/main/java/org/rocksdb/RateLimiterMode.java - src/main/java/org/rocksdb/ReadOptions.java - src/main/java/org/rocksdb/ReadTier.java - src/main/java/org/rocksdb/RemoveEmptyValueCompactionFilter.java - src/main/java/org/rocksdb/RestoreOptions.java - src/main/java/org/rocksdb/ReusedSynchronisationType.java - src/main/java/org/rocksdb/RocksCallbackObject.java - src/main/java/org/rocksdb/RocksDBException.java - src/main/java/org/rocksdb/RocksDB.java - src/main/java/org/rocksdb/RocksEnv.java - src/main/java/org/rocksdb/RocksIteratorInterface.java - src/main/java/org/rocksdb/RocksIterator.java - src/main/java/org/rocksdb/RocksMemEnv.java - src/main/java/org/rocksdb/RocksMutableObject.java - src/main/java/org/rocksdb/RocksObject.java - src/main/java/org/rocksdb/SanityLevel.java - src/main/java/org/rocksdb/SizeApproximationFlag.java - src/main/java/org/rocksdb/SkipListMemTableConfig.java - src/main/java/org/rocksdb/Slice.java - src/main/java/org/rocksdb/Snapshot.java - src/main/java/org/rocksdb/SstFileManager.java - src/main/java/org/rocksdb/SstFileMetaData.java - src/main/java/org/rocksdb/SstFileReader.java - src/main/java/org/rocksdb/SstFileReaderIterator.java - src/main/java/org/rocksdb/SstFileWriter.java - src/main/java/org/rocksdb/SstPartitionerFactory.java - src/main/java/org/rocksdb/SstPartitionerFixedPrefixFactory.java - src/main/java/org/rocksdb/StateType.java - src/main/java/org/rocksdb/StatisticsCollectorCallback.java - src/main/java/org/rocksdb/StatisticsCollector.java - src/main/java/org/rocksdb/Statistics.java - src/main/java/org/rocksdb/StatsCollectorInput.java - src/main/java/org/rocksdb/StatsLevel.java - src/main/java/org/rocksdb/Status.java - src/main/java/org/rocksdb/StringAppendOperator.java - src/main/java/org/rocksdb/TableFileCreationBriefInfo.java - src/main/java/org/rocksdb/TableFileCreationInfo.java - src/main/java/org/rocksdb/TableFileCreationReason.java - src/main/java/org/rocksdb/TableFileDeletionInfo.java - src/main/java/org/rocksdb/TableFilter.java - src/main/java/org/rocksdb/TableProperties.java - src/main/java/org/rocksdb/TableFormatConfig.java - src/main/java/org/rocksdb/ThreadType.java - src/main/java/org/rocksdb/ThreadStatus.java - src/main/java/org/rocksdb/TickerType.java - src/main/java/org/rocksdb/TimedEnv.java - src/main/java/org/rocksdb/TraceOptions.java - src/main/java/org/rocksdb/TraceWriter.java - src/main/java/org/rocksdb/TransactionalDB.java - src/main/java/org/rocksdb/TransactionalOptions.java - src/main/java/org/rocksdb/TransactionDB.java - src/main/java/org/rocksdb/TransactionDBOptions.java - src/main/java/org/rocksdb/Transaction.java - src/main/java/org/rocksdb/TransactionLogIterator.java - src/main/java/org/rocksdb/TransactionOptions.java - src/main/java/org/rocksdb/TtlDB.java - src/main/java/org/rocksdb/TxnDBWritePolicy.java - src/main/java/org/rocksdb/VectorMemTableConfig.java - src/main/java/org/rocksdb/WalFileType.java - src/main/java/org/rocksdb/WalFilter.java - src/main/java/org/rocksdb/WalProcessingOption.java - src/main/java/org/rocksdb/WALRecoveryMode.java - src/main/java/org/rocksdb/WBWIRocksIterator.java - src/main/java/org/rocksdb/WriteBatch.java - src/main/java/org/rocksdb/WriteBatchInterface.java - src/main/java/org/rocksdb/WriteBatchWithIndex.java - src/main/java/org/rocksdb/WriteOptions.java - src/main/java/org/rocksdb/WriteBufferManager.java - src/main/java/org/rocksdb/WriteStallCondition.java - src/main/java/org/rocksdb/WriteStallInfo.java - src/main/java/org/rocksdb/util/ByteUtil.java - src/main/java/org/rocksdb/util/BytewiseComparator.java - src/main/java/org/rocksdb/util/Environment.java - src/main/java/org/rocksdb/util/IntComparator.java - src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java - src/main/java/org/rocksdb/util/SizeUnit.java - src/main/java/org/rocksdb/UInt64AddOperator.java + src/main/java/org/forstdb/AbstractCompactionFilter.java + src/main/java/org/forstdb/AbstractCompactionFilterFactory.java + src/main/java/org/forstdb/AbstractComparator.java + src/main/java/org/forstdb/AbstractEventListener.java + src/main/java/org/forstdb/AbstractImmutableNativeReference.java + src/main/java/org/forstdb/AbstractMutableOptions.java + src/main/java/org/forstdb/AbstractNativeReference.java + src/main/java/org/forstdb/AbstractRocksIterator.java + src/main/java/org/forstdb/AbstractSlice.java + src/main/java/org/forstdb/AbstractTableFilter.java + src/main/java/org/forstdb/AbstractTraceWriter.java + src/main/java/org/forstdb/AbstractTransactionNotifier.java + src/main/java/org/forstdb/AbstractWalFilter.java + src/main/java/org/forstdb/AbstractWriteBatch.java + src/main/java/org/forstdb/AccessHint.java + src/main/java/org/forstdb/AdvancedColumnFamilyOptionsInterface.java + src/main/java/org/forstdb/AdvancedMutableColumnFamilyOptionsInterface.java + src/main/java/org/forstdb/BackgroundErrorReason.java + src/main/java/org/forstdb/BackupEngineOptions.java + src/main/java/org/forstdb/BackupEngine.java + src/main/java/org/forstdb/BackupInfo.java + src/main/java/org/forstdb/BlockBasedTableConfig.java + src/main/java/org/forstdb/BloomFilter.java + src/main/java/org/forstdb/BuiltinComparator.java + src/main/java/org/forstdb/ByteBufferGetStatus.java + src/main/java/org/forstdb/Cache.java + src/main/java/org/forstdb/CassandraCompactionFilter.java + src/main/java/org/forstdb/CassandraValueMergeOperator.java + src/main/java/org/forstdb/Checkpoint.java + src/main/java/org/forstdb/ChecksumType.java + src/main/java/org/forstdb/ClockCache.java + src/main/java/org/forstdb/ColumnFamilyDescriptor.java + src/main/java/org/forstdb/ColumnFamilyHandle.java + src/main/java/org/forstdb/ColumnFamilyMetaData.java + src/main/java/org/forstdb/ColumnFamilyOptionsInterface.java + src/main/java/org/forstdb/ColumnFamilyOptions.java + src/main/java/org/forstdb/CompactionJobInfo.java + src/main/java/org/forstdb/CompactionJobStats.java + src/main/java/org/forstdb/CompactionOptions.java + src/main/java/org/forstdb/CompactionOptionsFIFO.java + src/main/java/org/forstdb/CompactionOptionsUniversal.java + src/main/java/org/forstdb/CompactionPriority.java + src/main/java/org/forstdb/CompactionReason.java + src/main/java/org/forstdb/CompactRangeOptions.java + src/main/java/org/forstdb/CompactionStopStyle.java + src/main/java/org/forstdb/CompactionStyle.java + src/main/java/org/forstdb/ComparatorOptions.java + src/main/java/org/forstdb/ComparatorType.java + src/main/java/org/forstdb/CompressionOptions.java + src/main/java/org/forstdb/CompressionType.java + src/main/java/org/forstdb/ConfigOptions.java + src/main/java/org/forstdb/DataBlockIndexType.java + src/main/java/org/forstdb/DBOptionsInterface.java + src/main/java/org/forstdb/DBOptions.java + src/main/java/org/forstdb/DbPath.java + src/main/java/org/forstdb/DirectSlice.java + src/main/java/org/forstdb/EncodingType.java + src/main/java/org/forstdb/Env.java + src/main/java/org/forstdb/EnvFlinkTestSuite.java + src/main/java/org/forstdb/EnvOptions.java + src/main/java/org/forstdb/EventListener.java + src/main/java/org/forstdb/Experimental.java + src/main/java/org/forstdb/ExportImportFilesMetaData.java + src/main/java/org/forstdb/ExternalFileIngestionInfo.java + src/main/java/org/forstdb/Filter.java + src/main/java/org/forstdb/FilterPolicyType.java + src/main/java/org/forstdb/FileOperationInfo.java + src/main/java/org/forstdb/FlinkCompactionFilter.java + src/main/java/org/forstdb/FlinkEnv.java + src/main/java/org/forstdb/FlushJobInfo.java + src/main/java/org/forstdb/FlushReason.java + src/main/java/org/forstdb/FlushOptions.java + src/main/java/org/forstdb/GetStatus.java + src/main/java/org/forstdb/HashLinkedListMemTableConfig.java + src/main/java/org/forstdb/HashSkipListMemTableConfig.java + src/main/java/org/forstdb/HistogramData.java + src/main/java/org/forstdb/HistogramType.java + src/main/java/org/forstdb/Holder.java + src/main/java/org/forstdb/ImportColumnFamilyOptions.java + src/main/java/org/forstdb/HyperClockCache.java + src/main/java/org/forstdb/IndexShorteningMode.java + src/main/java/org/forstdb/IndexType.java + src/main/java/org/forstdb/InfoLogLevel.java + src/main/java/org/forstdb/IngestExternalFileOptions.java + src/main/java/org/forstdb/LevelMetaData.java + src/main/java/org/forstdb/ConcurrentTaskLimiter.java + src/main/java/org/forstdb/ConcurrentTaskLimiterImpl.java + src/main/java/org/forstdb/KeyMayExist.java + src/main/java/org/forstdb/LiveFileMetaData.java + src/main/java/org/forstdb/LogFile.java + src/main/java/org/forstdb/Logger.java + src/main/java/org/forstdb/LRUCache.java + src/main/java/org/forstdb/MemoryUsageType.java + src/main/java/org/forstdb/MemoryUtil.java + src/main/java/org/forstdb/MemTableConfig.java + src/main/java/org/forstdb/MemTableInfo.java + src/main/java/org/forstdb/MergeOperator.java + src/main/java/org/forstdb/MutableColumnFamilyOptions.java + src/main/java/org/forstdb/MutableColumnFamilyOptionsInterface.java + src/main/java/org/forstdb/MutableDBOptions.java + src/main/java/org/forstdb/MutableDBOptionsInterface.java + src/main/java/org/forstdb/MutableOptionKey.java + src/main/java/org/forstdb/MutableOptionValue.java + src/main/java/org/forstdb/NativeComparatorWrapper.java + src/main/java/org/forstdb/NativeLibraryLoader.java + src/main/java/org/forstdb/OperationStage.java + src/main/java/org/forstdb/OperationType.java + src/main/java/org/forstdb/OptimisticTransactionDB.java + src/main/java/org/forstdb/OptimisticTransactionOptions.java + src/main/java/org/forstdb/Options.java + src/main/java/org/forstdb/OptionString.java + src/main/java/org/forstdb/OptionsUtil.java + src/main/java/org/forstdb/PersistentCache.java + src/main/java/org/forstdb/PerfContext.java + src/main/java/org/forstdb/PerfLevel.java + src/main/java/org/forstdb/PlainTableConfig.java + src/main/java/org/forstdb/PrepopulateBlobCache.java + src/main/java/org/forstdb/Priority.java + src/main/java/org/forstdb/Range.java + src/main/java/org/forstdb/RateLimiter.java + src/main/java/org/forstdb/RateLimiterMode.java + src/main/java/org/forstdb/ReadOptions.java + src/main/java/org/forstdb/ReadTier.java + src/main/java/org/forstdb/RemoveEmptyValueCompactionFilter.java + src/main/java/org/forstdb/RestoreOptions.java + src/main/java/org/forstdb/ReusedSynchronisationType.java + src/main/java/org/forstdb/RocksCallbackObject.java + src/main/java/org/forstdb/RocksDBException.java + src/main/java/org/forstdb/RocksDB.java + src/main/java/org/forstdb/RocksEnv.java + src/main/java/org/forstdb/RocksIteratorInterface.java + src/main/java/org/forstdb/RocksIterator.java + src/main/java/org/forstdb/RocksMemEnv.java + src/main/java/org/forstdb/RocksMutableObject.java + src/main/java/org/forstdb/RocksObject.java + src/main/java/org/forstdb/SanityLevel.java + src/main/java/org/forstdb/SizeApproximationFlag.java + src/main/java/org/forstdb/SkipListMemTableConfig.java + src/main/java/org/forstdb/Slice.java + src/main/java/org/forstdb/Snapshot.java + src/main/java/org/forstdb/SstFileManager.java + src/main/java/org/forstdb/SstFileMetaData.java + src/main/java/org/forstdb/SstFileReader.java + src/main/java/org/forstdb/SstFileReaderIterator.java + src/main/java/org/forstdb/SstFileWriter.java + src/main/java/org/forstdb/SstPartitionerFactory.java + src/main/java/org/forstdb/SstPartitionerFixedPrefixFactory.java + src/main/java/org/forstdb/StateType.java + src/main/java/org/forstdb/StatisticsCollectorCallback.java + src/main/java/org/forstdb/StatisticsCollector.java + src/main/java/org/forstdb/Statistics.java + src/main/java/org/forstdb/StatsCollectorInput.java + src/main/java/org/forstdb/StatsLevel.java + src/main/java/org/forstdb/Status.java + src/main/java/org/forstdb/StringAppendOperator.java + src/main/java/org/forstdb/TableFileCreationBriefInfo.java + src/main/java/org/forstdb/TableFileCreationInfo.java + src/main/java/org/forstdb/TableFileCreationReason.java + src/main/java/org/forstdb/TableFileDeletionInfo.java + src/main/java/org/forstdb/TableFilter.java + src/main/java/org/forstdb/TableProperties.java + src/main/java/org/forstdb/TableFormatConfig.java + src/main/java/org/forstdb/ThreadType.java + src/main/java/org/forstdb/ThreadStatus.java + src/main/java/org/forstdb/TickerType.java + src/main/java/org/forstdb/TimedEnv.java + src/main/java/org/forstdb/TraceOptions.java + src/main/java/org/forstdb/TraceWriter.java + src/main/java/org/forstdb/TransactionalDB.java + src/main/java/org/forstdb/TransactionalOptions.java + src/main/java/org/forstdb/TransactionDB.java + src/main/java/org/forstdb/TransactionDBOptions.java + src/main/java/org/forstdb/Transaction.java + src/main/java/org/forstdb/TransactionLogIterator.java + src/main/java/org/forstdb/TransactionOptions.java + src/main/java/org/forstdb/TtlDB.java + src/main/java/org/forstdb/TxnDBWritePolicy.java + src/main/java/org/forstdb/VectorMemTableConfig.java + src/main/java/org/forstdb/WalFileType.java + src/main/java/org/forstdb/WalFilter.java + src/main/java/org/forstdb/WalProcessingOption.java + src/main/java/org/forstdb/WALRecoveryMode.java + src/main/java/org/forstdb/WBWIRocksIterator.java + src/main/java/org/forstdb/WriteBatch.java + src/main/java/org/forstdb/WriteBatchInterface.java + src/main/java/org/forstdb/WriteBatchWithIndex.java + src/main/java/org/forstdb/WriteOptions.java + src/main/java/org/forstdb/WriteBufferManager.java + src/main/java/org/forstdb/WriteStallCondition.java + src/main/java/org/forstdb/WriteStallInfo.java + src/main/java/org/forstdb/util/BufferUtil.java + src/main/java/org/forstdb/util/ByteUtil.java + src/main/java/org/forstdb/util/BytewiseComparator.java + src/main/java/org/forstdb/util/Environment.java + src/main/java/org/forstdb/util/IntComparator.java + src/main/java/org/forstdb/util/ReverseBytewiseComparator.java + src/main/java/org/forstdb/util/SizeUnit.java + src/main/java/org/forstdb/UInt64AddOperator.java + src/test/java/org/forstdb/NativeComparatorWrapperTest.java + src/test/java/org/forstdb/RocksDBExceptionTest.java + src/test/java/org/forstdb/test/TestableEventListener.java + src/test/java/org/forstdb/WriteBatchTest.java + src/test/java/org/forstdb/RocksNativeLibraryResource.java + src/test/java/org/forstdb/util/CapturingWriteBatchHandler.java + src/test/java/org/forstdb/util/WriteBatchGetter.java ) set(JAVA_TEST_CLASSES - src/test/java/org/rocksdb/BackupEngineTest.java - src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java - src/test/java/org/rocksdb/NativeComparatorWrapperTest.java - src/test/java/org/rocksdb/PlatformRandomHelper.java - src/test/java/org/rocksdb/RocksDBExceptionTest.java - src/test/java/org/rocksdb/RocksNativeLibraryResource.java - src/test/java/org/rocksdb/SnapshotTest.java - src/test/java/org/rocksdb/WriteBatchTest.java - src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java - src/test/java/org/rocksdb/util/WriteBatchGetter.java - src/test/java/org/rocksdb/test/TestableEventListener.java + src/test/java/org/forstdb/ConcurrentTaskLimiterTest.java + src/test/java/org/forstdb/EventListenerTest.java + src/test/java/org/forstdb/CompactionOptionsTest.java + src/test/java/org/forstdb/PlatformRandomHelper.java + src/test/java/org/forstdb/IngestExternalFileOptionsTest.java + src/test/java/org/forstdb/MutableDBOptionsTest.java + src/test/java/org/forstdb/WriteOptionsTest.java + src/test/java/org/forstdb/SstPartitionerTest.java + src/test/java/org/forstdb/RocksMemEnvTest.java + src/test/java/org/forstdb/CompactionOptionsUniversalTest.java + src/test/java/org/forstdb/ClockCacheTest.java + src/test/java/org/forstdb/BytewiseComparatorRegressionTest.java + src/test/java/org/forstdb/SnapshotTest.java + src/test/java/org/forstdb/CompactionJobStatsTest.java + src/test/java/org/forstdb/MemTableTest.java + src/test/java/org/forstdb/CompactionFilterFactoryTest.java + src/test/java/org/forstdb/DefaultEnvTest.java + src/test/java/org/forstdb/DBOptionsTest.java + src/test/java/org/forstdb/RocksIteratorTest.java + src/test/java/org/forstdb/SliceTest.java + src/test/java/org/forstdb/MultiGetTest.java + src/test/java/org/forstdb/ComparatorOptionsTest.java + src/test/java/org/forstdb/NativeLibraryLoaderTest.java + src/test/java/org/forstdb/StatisticsTest.java + src/test/java/org/forstdb/WALRecoveryModeTest.java + src/test/java/org/forstdb/TransactionLogIteratorTest.java + src/test/java/org/forstdb/ReadOptionsTest.java + src/test/java/org/forstdb/SecondaryDBTest.java + src/test/java/org/forstdb/KeyMayExistTest.java + src/test/java/org/forstdb/BlobOptionsTest.java + src/test/java/org/forstdb/InfoLogLevelTest.java + src/test/java/org/forstdb/CompactionPriorityTest.java + src/test/java/org/forstdb/FlushOptionsTest.java + src/test/java/org/forstdb/VerifyChecksumsTest.java + src/test/java/org/forstdb/MultiColumnRegressionTest.java + src/test/java/org/forstdb/FlushTest.java + src/test/java/org/forstdb/HyperClockCacheTest.java + src/test/java/org/forstdb/PutMultiplePartsTest.java + src/test/java/org/forstdb/StatisticsCollectorTest.java + src/test/java/org/forstdb/LRUCacheTest.java + src/test/java/org/forstdb/ColumnFamilyOptionsTest.java + src/test/java/org/forstdb/TransactionTest.java + src/test/java/org/forstdb/CompactionOptionsFIFOTest.java + src/test/java/org/forstdb/BackupEngineOptionsTest.java + src/test/java/org/forstdb/CheckPointTest.java + src/test/java/org/forstdb/PlainTableConfigTest.java + src/test/java/org/forstdb/TransactionDBOptionsTest.java + src/test/java/org/forstdb/ReadOnlyTest.java + src/test/java/org/forstdb/EnvOptionsTest.java + src/test/java/org/forstdb/test/RemoveEmptyValueCompactionFilterFactory.java + src/test/java/org/forstdb/test/RemoveEmptyValueCompactionFilterFactory.java + src/test/java/org/forstdb/test/TestableEventListener.java + src/test/java/org/forstdb/test/RemoveEmptyValueCompactionFilterFactory.java + src/test/java/org/forstdb/test/TestableEventListener.java + src/test/java/org/forstdb/test/RocksJunitRunner.java + src/test/java/org/forstdb/LoggerTest.java + src/test/java/org/forstdb/FilterTest.java + src/test/java/org/forstdb/ByteBufferUnsupportedOperationTest.java + src/test/java/org/forstdb/util/IntComparatorTest.java + src/test/java/org/forstdb/util/JNIComparatorTest.java + src/test/java/org/forstdb/util/ByteBufferAllocator.java + src/test/java/org/forstdb/util/SizeUnitTest.java + src/test/java/org/forstdb/util/BytewiseComparatorTest.java + src/test/java/org/forstdb/util/EnvironmentTest.java + src/test/java/org/forstdb/util/BytewiseComparatorIntTest.java + src/test/java/org/forstdb/util/DirectByteBufferAllocator.java + src/test/java/org/forstdb/util/HeapByteBufferAllocator.java + src/test/java/org/forstdb/util/TestUtil.java + src/test/java/org/forstdb/util/ReverseBytewiseComparatorIntTest.java + src/test/java/org/forstdb/Types.java + src/test/java/org/forstdb/MixedOptionsTest.java + src/test/java/org/forstdb/CompactRangeOptionsTest.java + src/test/java/org/forstdb/SstFileWriterTest.java + src/test/java/org/forstdb/WalFilterTest.java + src/test/java/org/forstdb/AbstractTransactionTest.java + src/test/java/org/forstdb/MergeTest.java + src/test/java/org/forstdb/OptionsTest.java + src/test/java/org/forstdb/WriteBatchThreadedTest.java + src/test/java/org/forstdb/MultiGetManyKeysTest.java + src/test/java/org/forstdb/TimedEnvTest.java + src/test/java/org/forstdb/CompactionStopStyleTest.java + src/test/java/org/forstdb/CompactionJobInfoTest.java + src/test/java/org/forstdb/BlockBasedTableConfigTest.java + src/test/java/org/forstdb/BuiltinComparatorTest.java + src/test/java/org/forstdb/RateLimiterTest.java + src/test/java/org/forstdb/TransactionOptionsTest.java + src/test/java/org/forstdb/WriteBatchWithIndexTest.java + src/test/java/org/forstdb/WriteBatchHandlerTest.java + src/test/java/org/forstdb/OptimisticTransactionDBTest.java + src/test/java/org/forstdb/OptionsUtilTest.java + src/test/java/org/forstdb/OptimisticTransactionTest.java + src/test/java/org/forstdb/MutableColumnFamilyOptionsTest.java + src/test/java/org/forstdb/CompressionOptionsTest.java + src/test/java/org/forstdb/ColumnFamilyTest.java + src/test/java/org/forstdb/SstFileReaderTest.java + src/test/java/org/forstdb/TransactionDBTest.java + src/test/java/org/forstdb/RocksDBTest.java + src/test/java/org/forstdb/MutableOptionsGetSetTest.java + src/test/java/org/forstdb/OptimisticTransactionOptionsTest.java + src/test/java/org/forstdb/SstFileManagerTest.java + src/test/java/org/forstdb/BackupEngineTest.java + src/test/java/org/forstdb/DirectSliceTest.java + src/test/java/org/forstdb/StatsCallbackMock.java + src/test/java/org/forstdb/CompressionTypesTest.java + src/test/java/org/forstdb/MemoryUtilTest.java + src/test/java/org/forstdb/TableFilterTest.java + src/test/java/org/forstdb/TtlDBTest.java +) + +set(JAVA_TEST_RUNNING_CLASSES + org.forstdb.ConcurrentTaskLimiterTest + org.forstdb.EventListenerTest + org.forstdb.CompactionOptionsTest + org.forstdb.IngestExternalFileOptionsTest + org.forstdb.MutableDBOptionsTest + org.forstdb.WriteOptionsTest + org.forstdb.SstPartitionerTest + org.forstdb.RocksMemEnvTest + org.forstdb.CompactionOptionsUniversalTest + org.forstdb.ClockCacheTest + # org.forstdb.BytewiseComparatorRegressionTest + org.forstdb.SnapshotTest + org.forstdb.CompactionJobStatsTest + org.forstdb.MemTableTest + org.forstdb.CompactionFilterFactoryTest + # org.forstdb.DefaultEnvTest + org.forstdb.DBOptionsTest + org.forstdb.WriteBatchTest + org.forstdb.RocksIteratorTest + org.forstdb.SliceTest + org.forstdb.MultiGetTest + org.forstdb.ComparatorOptionsTest + # org.forstdb.NativeLibraryLoaderTest + org.forstdb.StatisticsTest + org.forstdb.WALRecoveryModeTest + org.forstdb.TransactionLogIteratorTest + org.forstdb.ReadOptionsTest + org.forstdb.SecondaryDBTest + org.forstdb.KeyMayExistTest + org.forstdb.BlobOptionsTest + org.forstdb.InfoLogLevelTest + org.forstdb.CompactionPriorityTest + org.forstdb.FlushOptionsTest + org.forstdb.VerifyChecksumsTest + org.forstdb.MultiColumnRegressionTest + org.forstdb.FlushTest + org.forstdb.HyperClockCacheTest + org.forstdb.PutMultiplePartsTest + org.forstdb.StatisticsCollectorTest + org.forstdb.LRUCacheTest + org.forstdb.ColumnFamilyOptionsTest + org.forstdb.TransactionTest + org.forstdb.CompactionOptionsFIFOTest + org.forstdb.BackupEngineOptionsTest + org.forstdb.CheckPointTest + org.forstdb.PlainTableConfigTest + # org.forstdb.TransactionDBOptionsTest + org.forstdb.ReadOnlyTest + org.forstdb.EnvOptionsTest + org.forstdb.LoggerTest + org.forstdb.FilterTest + # org.forstdb.ByteBufferUnsupportedOperationTest + # org.forstdb.util.IntComparatorTest + # org.forstdb.util.JNIComparatorTest + org.forstdb.util.SizeUnitTest + # org.forstdb.util.BytewiseComparatorTest + org.forstdb.util.EnvironmentTest + # org.forstdb.util.BytewiseComparatorIntTest + # org.forstdb.util.ReverseBytewiseComparatorIntTest + org.forstdb.MixedOptionsTest + org.forstdb.CompactRangeOptionsTest + # org.forstdb.SstFileWriterTest + org.forstdb.WalFilterTest + # org.forstdb.AbstractTransactionTest + org.forstdb.MergeTest + org.forstdb.OptionsTest + org.forstdb.WriteBatchThreadedTest + org.forstdb.MultiGetManyKeysTest + org.forstdb.TimedEnvTest + org.forstdb.CompactionStopStyleTest + org.forstdb.CompactionJobInfoTest + org.forstdb.BlockBasedTableConfigTest + org.forstdb.BuiltinComparatorTest + org.forstdb.RateLimiterTest + # org.forstdb.TransactionOptionsTest + org.forstdb.WriteBatchWithIndexTest + org.forstdb.WriteBatchHandlerTest + org.forstdb.OptimisticTransactionDBTest + org.forstdb.OptionsUtilTest + org.forstdb.OptimisticTransactionTest + org.forstdb.MutableColumnFamilyOptionsTest + org.forstdb.CompressionOptionsTest + org.forstdb.ColumnFamilyTest + org.forstdb.SstFileReaderTest + org.forstdb.TransactionDBTest + org.forstdb.RocksDBTest + org.forstdb.MutableOptionsGetSetTest + # org.forstdb.OptimisticTransactionOptionsTest + org.forstdb.SstFileManagerTest + org.forstdb.BackupEngineTest + org.forstdb.DirectSliceTest + org.forstdb.CompressionTypesTest + org.forstdb.MemoryUtilTest + org.forstdb.TableFilterTest + org.forstdb.TtlDBTest ) include(FindJava) @@ -310,12 +526,20 @@ include_directories(${PROJECT_SOURCE_DIR}/java) set(JAVA_TEST_LIBDIR ${PROJECT_SOURCE_DIR}/java/test-libs) set(JAVA_TMP_JAR ${JAVA_TEST_LIBDIR}/tmp.jar) -set(JAVA_JUNIT_JAR ${JAVA_TEST_LIBDIR}/junit-4.12.jar) -set(JAVA_HAMCR_JAR ${JAVA_TEST_LIBDIR}/hamcrest-core-1.3.jar) -set(JAVA_MOCKITO_JAR ${JAVA_TEST_LIBDIR}/mockito-all-1.10.19.jar) -set(JAVA_CGLIB_JAR ${JAVA_TEST_LIBDIR}/cglib-2.2.2.jar) -set(JAVA_ASSERTJ_JAR ${JAVA_TEST_LIBDIR}/assertj-core-1.7.1.jar) +set(JAVA_JUNIT_JAR ${JAVA_TEST_LIBDIR}/junit-${JAVA_JUNIT_VERSION}.jar) +set(JAVA_HAMCR_JAR ${JAVA_TEST_LIBDIR}/hamcrest-${JAVA_HAMCR_VERSION}.jar) +set(JAVA_MOCKITO_JAR ${JAVA_TEST_LIBDIR}/mockito-all-${JAVA_MOCKITO_VERSION}.jar) +set(JAVA_CGLIB_JAR ${JAVA_TEST_LIBDIR}/cglib-${JAVA_CGLIB_VERSION}.jar) +set(JAVA_ASSERTJ_JAR ${JAVA_TEST_LIBDIR}/assertj-core-${JAVA_ASSERTJ_VERSION}.jar) set(JAVA_TESTCLASSPATH ${JAVA_JUNIT_JAR} ${JAVA_HAMCR_JAR} ${JAVA_MOCKITO_JAR} ${JAVA_CGLIB_JAR} ${JAVA_ASSERTJ_JAR}) +message("CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}") +message("MINGW: ${MINGW}") + +if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(JAVA_RUN_TESTCLASSPATH ${JAVA_JUNIT_JAR}$${JAVA_HAMCR_JAR}$${JAVA_MOCKITO_JAR}$${JAVA_CGLIB_JAR}$${JAVA_ASSERTJ_JAR}) +else() + set(JAVA_RUN_TESTCLASSPATH ${JAVA_JUNIT_JAR}:${JAVA_HAMCR_JAR}:${JAVA_MOCKITO_JAR}:${JAVA_CGLIB_JAR}:${JAVA_ASSERTJ_JAR}) +endif() set(JNI_OUTPUT_DIR ${PROJECT_SOURCE_DIR}/java/include) file(MAKE_DIRECTORY ${JNI_OUTPUT_DIR}) @@ -333,26 +557,31 @@ elseif(${CMAKE_VERSION} VERSION_LESS "3.11.4") message("Using an old CMAKE (${CMAKE_VERSION}) - JNI headers generated in separate step") add_jar( rocksdbjni_classes - SOURCES - ${JAVA_MAIN_CLASSES} - ${JAVA_TEST_CLASSES} - INCLUDE_JARS ${JAVA_TESTCLASSPATH} + SOURCES ${JAVA_MAIN_CLASSES} ) else () # Java 1.8 or newer prepare the JAR... message("Preparing Jar for JDK ${Java_VERSION_STRING}") + message("JAVA_TESTCLASSPATH=${JAVA_TESTCLASSPATH}") add_jar( rocksdbjni_classes - SOURCES - ${JAVA_MAIN_CLASSES} - ${JAVA_TEST_CLASSES} - INCLUDE_JARS ${JAVA_TESTCLASSPATH} + SOURCES ${JAVA_MAIN_CLASSES} + INCLUDE_JARS ${ROCKSDBJNI_CLASSES_JAR_FILE} ${JAVA_TESTCLASSPATH} GENERATE_NATIVE_HEADERS rocksdbjni_headers DESTINATION ${JNI_OUTPUT_DIR} ) endif() +add_jar( + rocksdbjni_test_classes + SOURCES + ${JAVA_MAIN_CLASSES} + ${JAVA_TEST_CLASSES} + INCLUDE_JARS ${JAVA_TESTCLASSPATH} + GENERATE_NATIVE_HEADERS rocksdbjni_test_headers DESTINATION ${JNI_OUTPUT_DIR} +) + if(NOT EXISTS ${PROJECT_SOURCE_DIR}/java/classes) file(MAKE_DIRECTORY ${PROJECT_SOURCE_DIR}/java/classes) endif() @@ -371,7 +600,7 @@ endif() if(NOT EXISTS ${JAVA_JUNIT_JAR}) message("Downloading ${JAVA_JUNIT_JAR}") - file(DOWNLOAD ${DEPS_URL}/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/junit-${JAVA_JUNIT_VERSION}.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) list(GET downloadStatus 1 error_message) if(NOT error_code EQUAL 0) @@ -381,7 +610,7 @@ if(NOT EXISTS ${JAVA_JUNIT_JAR}) endif() if(NOT EXISTS ${JAVA_HAMCR_JAR}) message("Downloading ${JAVA_HAMCR_JAR}") - file(DOWNLOAD ${DEPS_URL}/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/hamcrest-${JAVA_HAMCR_VERSION}.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) list(GET downloadStatus 1 error_message) if(NOT error_code EQUAL 0) @@ -391,7 +620,7 @@ if(NOT EXISTS ${JAVA_HAMCR_JAR}) endif() if(NOT EXISTS ${JAVA_MOCKITO_JAR}) message("Downloading ${JAVA_MOCKITO_JAR}") - file(DOWNLOAD ${DEPS_URL}/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/mockito-all-${JAVA_MOCKITO_VERSION}.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) list(GET downloadStatus 1 error_message) if(NOT error_code EQUAL 0) @@ -401,7 +630,7 @@ if(NOT EXISTS ${JAVA_MOCKITO_JAR}) endif() if(NOT EXISTS ${JAVA_CGLIB_JAR}) message("Downloading ${JAVA_CGLIB_JAR}") - file(DOWNLOAD ${DEPS_URL}/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/cglib-${JAVA_CGLIB_VERSION}.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) list(GET downloadStatus 1 error_message) if(NOT error_code EQUAL 0) @@ -411,7 +640,7 @@ if(NOT EXISTS ${JAVA_CGLIB_JAR}) endif() if(NOT EXISTS ${JAVA_ASSERTJ_JAR}) message("Downloading ${JAVA_ASSERTJ_JAR}") - file(DOWNLOAD ${DEPS_URL}/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/assertj-core-${JAVA_ASSERTJ_VERSION}.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) list(GET downloadStatus 1 error_message) if(NOT error_code EQUAL 0) @@ -424,110 +653,111 @@ if(${CMAKE_VERSION} VERSION_LESS "3.11.4") # Old CMake ONLY generate JNI headers, otherwise JNI is handled in add_jar step above message("Preparing JNI headers for old CMake (${CMAKE_VERSION})") set(NATIVE_JAVA_CLASSES - org.rocksdb.AbstractCompactionFilter - org.rocksdb.AbstractCompactionFilterFactory - org.rocksdb.AbstractComparator - org.rocksdb.AbstractEventListener - org.rocksdb.AbstractImmutableNativeReference - org.rocksdb.AbstractNativeReference - org.rocksdb.AbstractRocksIterator - org.rocksdb.AbstractSlice - org.rocksdb.AbstractTableFilter - org.rocksdb.AbstractTraceWriter - org.rocksdb.AbstractTransactionNotifier - org.rocksdb.AbstractWalFilter - org.rocksdb.BackupEngineOptions - org.rocksdb.BackupEngine - org.rocksdb.BlockBasedTableConfig - org.rocksdb.BloomFilter - org.rocksdb.CassandraCompactionFilter - org.rocksdb.CassandraValueMergeOperator - org.rocksdb.Checkpoint - org.rocksdb.ClockCache - org.rocksdb.Cache - org.rocksdb.ColumnFamilyHandle - org.rocksdb.ColumnFamilyOptions - org.rocksdb.CompactionJobInfo - org.rocksdb.CompactionJobStats - org.rocksdb.CompactionOptions - org.rocksdb.CompactionOptionsFIFO - org.rocksdb.CompactionOptionsUniversal - org.rocksdb.CompactRangeOptions - org.rocksdb.ComparatorOptions - org.rocksdb.CompressionOptions - org.rocksdb.ConcurrentTaskLimiterImpl - org.rocksdb.ConfigOptions - org.rocksdb.DBOptions - org.rocksdb.DirectSlice - org.rocksdb.Env - org.rocksdb.EnvFlinkTestSuite - org.rocksdb.EnvOptions - org.rocksdb.Filter - org.rocksdb.FlinkCompactionFilter - org.rocksdb.FlinkEnv - org.rocksdb.FlushOptions - org.rocksdb.HashLinkedListMemTableConfig - org.rocksdb.HashSkipListMemTableConfig - org.rocksdb.IngestExternalFileOptions - org.rocksdb.Logger - org.rocksdb.LRUCache - org.rocksdb.MemoryUtil - org.rocksdb.MemTableConfig - org.rocksdb.NativeComparatorWrapper - org.rocksdb.NativeLibraryLoader - org.rocksdb.OptimisticTransactionDB - org.rocksdb.OptimisticTransactionOptions - org.rocksdb.Options - org.rocksdb.OptionsUtil - org.rocksdb.PersistentCache - org.rocksdb.PlainTableConfig - org.rocksdb.RateLimiter - org.rocksdb.ReadOptions - org.rocksdb.RemoveEmptyValueCompactionFilter - org.rocksdb.RestoreOptions - org.rocksdb.RocksCallbackObject - org.rocksdb.RocksDB - org.rocksdb.RocksEnv - org.rocksdb.RocksIterator - org.rocksdb.RocksIteratorInterface - org.rocksdb.RocksMemEnv - org.rocksdb.RocksMutableObject - org.rocksdb.RocksObject - org.rocksdb.SkipListMemTableConfig - org.rocksdb.Slice - org.rocksdb.Snapshot - org.rocksdb.SstFileManager - org.rocksdb.SstFileWriter - org.rocksdb.SstFileReader - org.rocksdb.SstFileReaderIterator - org.rocksdb.SstPartitionerFactory - org.rocksdb.SstPartitionerFixedPrefixFactory - org.rocksdb.Statistics - org.rocksdb.StringAppendOperator - org.rocksdb.TableFormatConfig - org.rocksdb.ThreadStatus - org.rocksdb.TimedEnv - org.rocksdb.Transaction - org.rocksdb.TransactionDB - org.rocksdb.TransactionDBOptions - org.rocksdb.TransactionLogIterator - org.rocksdb.TransactionOptions - org.rocksdb.TtlDB - org.rocksdb.UInt64AddOperator - org.rocksdb.VectorMemTableConfig - org.rocksdb.WBWIRocksIterator - org.rocksdb.WriteBatch - org.rocksdb.WriteBatch.Handler - org.rocksdb.WriteBatchInterface - org.rocksdb.WriteBatchWithIndex - org.rocksdb.WriteOptions - org.rocksdb.NativeComparatorWrapperTest - org.rocksdb.RocksDBExceptionTest - org.rocksdb.SnapshotTest - org.rocksdb.WriteBatchTest - org.rocksdb.WriteBatchTestInternalHelper - org.rocksdb.WriteBufferManager - org.rocksdb.test.TestableEventListener + org.forstdb.AbstractCompactionFilter + org.forstdb.AbstractCompactionFilterFactory + org.forstdb.AbstractComparator + org.forstdb.AbstractEventListener + org.forstdb.AbstractImmutableNativeReference + org.forstdb.AbstractNativeReference + org.forstdb.AbstractRocksIterator + org.forstdb.AbstractSlice + org.forstdb.AbstractTableFilter + org.forstdb.AbstractTraceWriter + org.forstdb.AbstractTransactionNotifier + org.forstdb.AbstractWalFilter + org.forstdb.BackupEngineOptions + org.forstdb.BackupEngine + org.forstdb.BlockBasedTableConfig + org.forstdb.BloomFilter + org.forstdb.CassandraCompactionFilter + org.forstdb.CassandraValueMergeOperator + org.forstdb.Checkpoint + org.forstdb.ClockCache + org.forstdb.Cache + org.forstdb.ColumnFamilyHandle + org.forstdb.ColumnFamilyOptions + org.forstdb.CompactionJobInfo + org.forstdb.CompactionJobStats + org.forstdb.CompactionOptions + org.forstdb.CompactionOptionsFIFO + org.forstdb.CompactionOptionsUniversal + org.forstdb.CompactRangeOptions + org.forstdb.ComparatorOptions + org.forstdb.CompressionOptions + org.forstdb.ConcurrentTaskLimiterImpl + org.forstdb.ConfigOptions + org.forstdb.DBOptions + org.forstdb.DirectSlice + org.forstdb.Env + org.forstdb.EnvFlinkTestSuite + org.forstdb.EnvOptions + org.forstdb.Filter + org.forstdb.FlinkCompactionFilter + org.forstdb.FlinkEnv + org.forstdb.FlushOptions + org.forstdb.HashLinkedListMemTableConfig + org.forstdb.HashSkipListMemTableConfig + org.forstdb.HyperClockCache + org.forstdb.IngestExternalFileOptions + org.forstdb.Logger + org.forstdb.LRUCache + org.forstdb.MemoryUtil + org.forstdb.MemTableConfig + org.forstdb.NativeComparatorWrapper + org.forstdb.NativeLibraryLoader + org.forstdb.OptimisticTransactionDB + org.forstdb.OptimisticTransactionOptions + org.forstdb.Options + org.forstdb.OptionsUtil + org.forstdb.PersistentCache + org.forstdb.PlainTableConfig + org.forstdb.RateLimiter + org.forstdb.ReadOptions + org.forstdb.RemoveEmptyValueCompactionFilter + org.forstdb.RestoreOptions + org.forstdb.RocksCallbackObject + org.forstdb.RocksDB + org.forstdb.RocksEnv + org.forstdb.RocksIterator + org.forstdb.RocksIteratorInterface + org.forstdb.RocksMemEnv + org.forstdb.RocksMutableObject + org.forstdb.RocksObject + org.forstdb.SkipListMemTableConfig + org.forstdb.Slice + org.forstdb.Snapshot + org.forstdb.SstFileManager + org.forstdb.SstFileWriter + org.forstdb.SstFileReader + org.forstdb.SstFileReaderIterator + org.forstdb.SstPartitionerFactory + org.forstdb.SstPartitionerFixedPrefixFactory + org.forstdb.Statistics + org.forstdb.StringAppendOperator + org.forstdb.TableFormatConfig + org.forstdb.ThreadStatus + org.forstdb.TimedEnv + org.forstdb.Transaction + org.forstdb.TransactionDB + org.forstdb.TransactionDBOptions + org.forstdb.TransactionLogIterator + org.forstdb.TransactionOptions + org.forstdb.TtlDB + org.forstdb.UInt64AddOperator + org.forstdb.VectorMemTableConfig + org.forstdb.WBWIRocksIterator + org.forstdb.WriteBatch + org.forstdb.WriteBatch.Handler + org.forstdb.WriteBatchInterface + org.forstdb.WriteBatchWithIndex + org.forstdb.WriteOptions + org.forstdb.NativeComparatorWrapperTest + org.forstdb.RocksDBExceptionTest + org.forstdb.SnapshotTest + org.forstdb.WriteBatchTest + org.forstdb.WriteBatchTestInternalHelper + org.forstdb.WriteBufferManager + org.forstdb.test.TestableEventListener ) create_javah( @@ -543,7 +773,12 @@ if(NOT MSVC) endif() set(ROCKSDBJNI_STATIC_LIB rocksdbjni${ARTIFACT_SUFFIX}) -add_library(${ROCKSDBJNI_STATIC_LIB} ${JNI_NATIVE_SOURCES}) +if(MINGW) + # Build mingw only as staic library. + add_library(${ROCKSDBJNI_STATIC_LIB} ${JNI_NATIVE_SOURCES}) +else() + add_library(${ROCKSDBJNI_STATIC_LIB} SHARED ${JNI_NATIVE_SOURCES}) +endif() add_dependencies(${ROCKSDBJNI_STATIC_LIB} rocksdbjni_headers) target_link_libraries(${ROCKSDBJNI_STATIC_LIB} ${ROCKSDB_STATIC_LIB} ${ROCKSDB_LIB}) @@ -560,3 +795,19 @@ if(NOT MINGW) COMPILE_PDB_NAME ${ROCKSDBJNI_STATIC_LIB}.pdb ) endif() + +enable_testing() +get_target_property(ROCKSDBJNI_CLASSES_TEST_JAR_FILE rocksdbjni_test_classes JAR_FILE) +foreach (CLAZZ ${JAVA_TEST_RUNNING_CLASSES}) + if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + add_test( + NAME jtest_${CLAZZ} + COMMAND ${Java_JAVA_EXECUTABLE} ${JVMARGS} -ea -Xcheck:jni -Djava.library.path=${PROJECT_BINARY_DIR}/java/${CMAKE_BUILD_TYPE} -classpath ${JAVA_RUN_TESTCLASSPATH}$${ROCKSDBJNI_CLASSES_TEST_JAR_FILE} org.forstdb.test.RocksJunitRunner ${CLAZZ} + ) + else() + add_test( + NAME jtest_${CLAZZ} + COMMAND ${Java_JAVA_EXECUTABLE} ${JVMARGS} -ea -Xcheck:jni -Djava.library.path=${PROJECT_BINARY_DIR}/java -classpath ${JAVA_RUN_TESTCLASSPATH}:${ROCKSDBJNI_CLASSES_TEST_JAR_FILE} org.forstdb.test.RocksJunitRunner ${CLAZZ} + ) + endif() +endforeach(CLAZZ) \ No newline at end of file diff --git a/java/GetBenchmarks.md b/java/GetPutBenchmarks.md similarity index 66% rename from java/GetBenchmarks.md rename to java/GetPutBenchmarks.md index b66a897e2..600b6377c 100644 --- a/java/GetBenchmarks.md +++ b/java/GetPutBenchmarks.md @@ -8,16 +8,16 @@ Mac ``` make clean jclean DEBUG_LEVEL=0 make -j12 rocksdbjava -(cd java/target; cp rocksdbjni-7.9.0-osx.jar rocksdbjni-7.9.0-SNAPSHOT-osx.jar) -mvn install:install-file -Dfile=./java/target/rocksdbjni-7.9.0-SNAPSHOT-osx.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=7.9.0-SNAPSHOT -Dpackaging=jar +(cd java/target; cp rocksdbjni-7.10.0-osx.jar rocksdbjni-7.10.0-SNAPSHOT-osx.jar) +mvn install:install-file -Dfile=./java/target/rocksdbjni-7.10.0-SNAPSHOT-osx.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=7.10.0-SNAPSHOT -Dpackaging=jar ``` Linux ``` make clean jclean DEBUG_LEVEL=0 make -j12 rocksdbjava -(cd java/target; cp rocksdbjni-7.9.0-linux64.jar rocksdbjni-7.9.0-SNAPSHOT-linux64.jar) -mvn install:install-file -Dfile=./java/target/rocksdbjni-7.9.0-SNAPSHOT-linux64.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=7.9.0-SNAPSHOT -Dpackaging=jar +(cd java/target; cp rocksdbjni-7.10.0-linux64.jar rocksdbjni-7.10.0-SNAPSHOT-linux64.jar) +mvn install:install-file -Dfile=./java/target/rocksdbjni-7.10.0-SNAPSHOT-linux64.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=7.10.0-SNAPSHOT -Dpackaging=jar ``` Build jmh test package, on either platform @@ -35,31 +35,10 @@ The long performance run (as big as we can make it on our Ubuntu box without fil java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000,50000 -p keySize=128 -p valueSize=1024,16384 -p columnFamilyTestType="1_column_family","20_column_families" GetBenchmarks.get GetBenchmarks.preallocatedByteBufferGet GetBenchmarks.preallocatedGet ``` -## Results (small runs, Mac) - -These are run on a 10-core M1 with 64GB of memory and 2TB of SSD. -They probably reflect the absolute best case for this optimization, hitting in-memory buffers and completely eliminating a buffer copy. - -### Before -Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (valueSize) Mode Cnt Score Error Units -GetBenchmarks.get no_column_family 1000 128 N/A 32768 thrpt 25 43496.578 ± 5743.090 ops/s -GetBenchmarks.preallocatedByteBufferGet no_column_family 1000 128 N/A 32768 thrpt 25 70765.578 ± 697.548 ops/s -GetBenchmarks.preallocatedGet no_column_family 1000 128 N/A 32768 thrpt 25 69883.554 ± 944.184 ops/s - -### After fixing byte[] (.get and .preallocatedGet) - -Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (valueSize) Mode Cnt Score Error Units -GetBenchmarks.get no_column_family 1000 128 N/A 32768 thrpt 25 149207.681 ± 2261.671 ops/s -GetBenchmarks.preallocatedByteBufferGet no_column_family 1000 128 N/A 32768 thrpt 25 68920.489 ± 1574.664 ops/s -GetBenchmarks.preallocatedGet no_column_family 1000 128 N/A 32768 thrpt 25 177399.022 ± 2107.375 ops/s +## Results (Ubuntu, big runs) -### After fixing ByteBuffer (.preallocatedByteBufferGet) +NB - we have removed some test results we initially observed on Mac which were not later reproducible. -Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (valueSize) Mode Cnt Score Error Units -GetBenchmarks.get no_column_family 1000 128 N/A 32768 thrpt 25 150389.259 ± 1371.473 ops/s -GetBenchmarks.preallocatedByteBufferGet no_column_family 1000 128 N/A 32768 thrpt 25 179919.468 ± 1670.714 ops/s -GetBenchmarks.preallocatedGet no_column_family 1000 128 N/A 32768 thrpt 25 178261.938 ± 2630.571 ops/s -## Results (Ubuntu, big runs) These take 3-4 hours ``` java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000,50000 -p keySize=128 -p valueSize=1024,16384 -p columnFamilyTestType="1_column_family","20_column_families" GetBenchmarks.get GetBenchmarks.preallocatedByteBufferGet GetBenchmarks.preallocatedGet @@ -67,6 +46,13 @@ java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000,500 It's clear that all `get()` variants have noticeably improved performance, though not the spectacular gains of the M1. ### With fixes for all of the `get()` instances +The tests which use methods which have had performance improvements applied are: +```java +get() +preallocatedGet() +preallocatedByteBufferGet() +``` + Benchmark (columnFamilyTestType) (keyCount) (keySize) (valueSize) Mode Cnt Score Error Units GetBenchmarks.get 1_column_family 1000 128 1024 thrpt 25 935648.793 ± 22879.910 ops/s GetBenchmarks.get 1_column_family 1000 128 16384 thrpt 25 204366.301 ± 1326.570 ops/s @@ -159,3 +145,60 @@ GetBenchmarks.preallocatedGet no_column_families 1000 The performance improvement is real. +# Put Performance Benchmarks + +Results associated with [Java API consistency between RocksDB.put() , .merge() and Transaction.put() , .merge()](https://github.com/facebook/rocksdb/pull/11019) + +This work was not designed specifically as a performance optimization, but we want to confirm that it has not regressed what it has changed, and to provide +a baseline for future possible performance work. + +## Build/Run + +Building is as above. Running is a different invocation of the same JMH jar. +``` +java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000,50000 -p keySize=128 -p valueSize=1024,32768 -p columnFamilyTestType="no_column_family" PutBenchmarks +``` + +## Before Changes + +These results were generated in a private branch with the `PutBenchmarks` from the PR backported onto the current *main*. + +Benchmark (bufferListSize) (columnFamilyTestType) (keyCount) (keySize) (valueSize) Mode Cnt Score Error Units +PutBenchmarks.put 16 no_column_family 1000 128 1024 thrpt 25 76670.200 ± 2555.248 ops/s +PutBenchmarks.put 16 no_column_family 1000 128 32768 thrpt 25 3913.692 ± 225.690 ops/s +PutBenchmarks.put 16 no_column_family 50000 128 1024 thrpt 25 74479.589 ± 988.361 ops/s +PutBenchmarks.put 16 no_column_family 50000 128 32768 thrpt 25 4070.800 ± 194.838 ops/s +PutBenchmarks.putByteArrays 16 no_column_family 1000 128 1024 thrpt 25 72150.853 ± 1744.216 ops/s +PutBenchmarks.putByteArrays 16 no_column_family 1000 128 32768 thrpt 25 3896.646 ± 188.629 ops/s +PutBenchmarks.putByteArrays 16 no_column_family 50000 128 1024 thrpt 25 71753.287 ± 1053.904 ops/s +PutBenchmarks.putByteArrays 16 no_column_family 50000 128 32768 thrpt 25 3928.503 ± 264.443 ops/s +PutBenchmarks.putByteBuffers 16 no_column_family 1000 128 1024 thrpt 25 72595.105 ± 1027.258 ops/s +PutBenchmarks.putByteBuffers 16 no_column_family 1000 128 32768 thrpt 25 3890.100 ± 199.131 ops/s +PutBenchmarks.putByteBuffers 16 no_column_family 50000 128 1024 thrpt 25 70878.133 ± 1181.601 ops/s +PutBenchmarks.putByteBuffers 16 no_column_family 50000 128 32768 thrpt 25 3863.181 ± 215.888 ops/s + +## After Changes + +These results were generated on the PR branch. + +Benchmark (bufferListSize) (columnFamilyTestType) (keyCount) (keySize) (valueSize) Mode Cnt Score Error Units +PutBenchmarks.put 16 no_column_family 1000 128 1024 thrpt 25 75178.751 ± 2644.775 ops/s +PutBenchmarks.put 16 no_column_family 1000 128 32768 thrpt 25 3937.175 ± 257.039 ops/s +PutBenchmarks.put 16 no_column_family 50000 128 1024 thrpt 25 74375.519 ± 1776.654 ops/s +PutBenchmarks.put 16 no_column_family 50000 128 32768 thrpt 25 4013.413 ± 257.706 ops/s +PutBenchmarks.putByteArrays 16 no_column_family 1000 128 1024 thrpt 25 71418.303 ± 1610.977 ops/s +PutBenchmarks.putByteArrays 16 no_column_family 1000 128 32768 thrpt 25 4027.581 ± 227.900 ops/s +PutBenchmarks.putByteArrays 16 no_column_family 50000 128 1024 thrpt 25 71229.107 ± 2720.083 ops/s +PutBenchmarks.putByteArrays 16 no_column_family 50000 128 32768 thrpt 25 4022.635 ± 212.540 ops/s +PutBenchmarks.putByteBuffers 16 no_column_family 1000 128 1024 thrpt 25 71718.501 ± 787.537 ops/s +PutBenchmarks.putByteBuffers 16 no_column_family 1000 128 32768 thrpt 25 4078.050 ± 176.331 ops/s +PutBenchmarks.putByteBuffers 16 no_column_family 50000 128 1024 thrpt 25 72736.754 ± 828.971 ops/s +PutBenchmarks.putByteBuffers 16 no_column_family 50000 128 32768 thrpt 25 3987.232 ± 205.577 ops/s + +## Discussion + +The changes don't appear to have had a material effect on performance. We are happy with this. + + * We would obviously advise running future changes before and after to confirm they have no adverse effects. + + diff --git a/java/Makefile b/java/Makefile index 0eb1cb36b..a73288a85 100644 --- a/java/Makefile +++ b/java/Makefile @@ -1,101 +1,103 @@ NATIVE_JAVA_CLASSES = \ - org.rocksdb.AbstractCompactionFilter\ - org.rocksdb.AbstractCompactionFilterFactory\ - org.rocksdb.AbstractComparator\ - org.rocksdb.AbstractEventListener\ - org.rocksdb.AbstractSlice\ - org.rocksdb.AbstractTableFilter\ - org.rocksdb.AbstractTraceWriter\ - org.rocksdb.AbstractTransactionNotifier\ - org.rocksdb.AbstractWalFilter\ - org.rocksdb.BackupEngine\ - org.rocksdb.BackupEngineOptions\ - org.rocksdb.BlockBasedTableConfig\ - org.rocksdb.BloomFilter\ - org.rocksdb.Checkpoint\ - org.rocksdb.ClockCache\ - org.rocksdb.Cache\ - org.rocksdb.CassandraCompactionFilter\ - org.rocksdb.CassandraValueMergeOperator\ - org.rocksdb.ColumnFamilyHandle\ - org.rocksdb.ColumnFamilyOptions\ - org.rocksdb.CompactionJobInfo\ - org.rocksdb.CompactionJobStats\ - org.rocksdb.CompactionOptions\ - org.rocksdb.CompactionOptionsFIFO\ - org.rocksdb.CompactionOptionsUniversal\ - org.rocksdb.CompactRangeOptions\ - org.rocksdb.ComparatorOptions\ - org.rocksdb.CompressionOptions\ - org.rocksdb.ConfigOptions\ - org.rocksdb.DBOptions\ - org.rocksdb.DirectSlice\ - org.rocksdb.Env\ - org.rocksdb.EnvOptions\ - org.rocksdb.FlinkCompactionFilter\ - org.rocksdb.FlushOptions\ - org.rocksdb.Filter\ - org.rocksdb.IngestExternalFileOptions\ - org.rocksdb.HashLinkedListMemTableConfig\ - org.rocksdb.HashSkipListMemTableConfig\ - org.rocksdb.ConcurrentTaskLimiter\ - org.rocksdb.ConcurrentTaskLimiterImpl\ - org.rocksdb.KeyMayExist\ - org.rocksdb.Logger\ - org.rocksdb.LRUCache\ - org.rocksdb.MemoryUsageType\ - org.rocksdb.MemoryUtil\ - org.rocksdb.MergeOperator\ - org.rocksdb.NativeComparatorWrapper\ - org.rocksdb.OptimisticTransactionDB\ - org.rocksdb.OptimisticTransactionOptions\ - org.rocksdb.Options\ - org.rocksdb.OptionsUtil\ - org.rocksdb.PersistentCache\ - org.rocksdb.PlainTableConfig\ - org.rocksdb.RateLimiter\ - org.rocksdb.ReadOptions\ - org.rocksdb.RemoveEmptyValueCompactionFilter\ - org.rocksdb.RestoreOptions\ - org.rocksdb.RocksCallbackObject\ - org.rocksdb.RocksDB\ - org.rocksdb.RocksEnv\ - org.rocksdb.RocksIterator\ - org.rocksdb.RocksMemEnv\ - org.rocksdb.SkipListMemTableConfig\ - org.rocksdb.Slice\ - org.rocksdb.SstFileManager\ - org.rocksdb.SstFileWriter\ - org.rocksdb.SstFileReader\ - org.rocksdb.SstFileReaderIterator\ - org.rocksdb.SstPartitionerFactory\ - org.rocksdb.SstPartitionerFixedPrefixFactory\ - org.rocksdb.Statistics\ - org.rocksdb.ThreadStatus\ - org.rocksdb.TimedEnv\ - org.rocksdb.Transaction\ - org.rocksdb.TransactionDB\ - org.rocksdb.TransactionDBOptions\ - org.rocksdb.TransactionOptions\ - org.rocksdb.TransactionLogIterator\ - org.rocksdb.TtlDB\ - org.rocksdb.VectorMemTableConfig\ - org.rocksdb.Snapshot\ - org.rocksdb.StringAppendOperator\ - org.rocksdb.UInt64AddOperator\ - org.rocksdb.WriteBatch\ - org.rocksdb.WriteBatch.Handler\ - org.rocksdb.WriteOptions\ - org.rocksdb.WriteBatchWithIndex\ - org.rocksdb.WriteBufferManager\ - org.rocksdb.WBWIRocksIterator + org.forstdb.AbstractCompactionFilter\ + org.forstdb.AbstractCompactionFilterFactory\ + org.forstdb.AbstractComparator\ + org.forstdb.AbstractEventListener\ + org.forstdb.AbstractSlice\ + org.forstdb.AbstractTableFilter\ + org.forstdb.AbstractTraceWriter\ + org.forstdb.AbstractTransactionNotifier\ + org.forstdb.AbstractWalFilter\ + org.forstdb.BackupEngine\ + org.forstdb.BackupEngineOptions\ + org.forstdb.BlockBasedTableConfig\ + org.forstdb.BloomFilter\ + org.forstdb.Checkpoint\ + org.forstdb.ClockCache\ + org.forstdb.Cache\ + org.forstdb.CassandraCompactionFilter\ + org.forstdb.CassandraValueMergeOperator\ + org.forstdb.ColumnFamilyHandle\ + org.forstdb.ColumnFamilyOptions\ + org.forstdb.CompactionJobInfo\ + org.forstdb.CompactionJobStats\ + org.forstdb.CompactionOptions\ + org.forstdb.CompactionOptionsFIFO\ + org.forstdb.CompactionOptionsUniversal\ + org.forstdb.CompactRangeOptions\ + org.forstdb.ComparatorOptions\ + org.forstdb.CompressionOptions\ + org.forstdb.ConfigOptions\ + org.forstdb.DBOptions\ + org.forstdb.DirectSlice\ + org.forstdb.Env\ + org.forstdb.EnvOptions\ + org.forstdb.FlinkCompactionFilter\ + org.forstdb.FlushOptions\ + org.forstdb.Filter\ + org.forstdb.IngestExternalFileOptions\ + org.forstdb.HashLinkedListMemTableConfig\ + org.forstdb.HashSkipListMemTableConfig\ + org.forstdb.ConcurrentTaskLimiter\ + org.forstdb.ConcurrentTaskLimiterImpl\ + org.forstdb.KeyMayExist\ + org.forstdb.Logger\ + org.forstdb.LRUCache\ + org.forstdb.MemoryUsageType\ + org.forstdb.MemoryUtil\ + org.forstdb.MergeOperator\ + org.forstdb.NativeComparatorWrapper\ + org.forstdb.OptimisticTransactionDB\ + org.forstdb.OptimisticTransactionOptions\ + org.forstdb.Options\ + org.forstdb.OptionsUtil\ + org.forstdb.PersistentCache\ + org.forstdb.PerfContext\ + org.forstdb.PerfLevel\ + org.forstdb.PlainTableConfig\ + org.forstdb.RateLimiter\ + org.forstdb.ReadOptions\ + org.forstdb.RemoveEmptyValueCompactionFilter\ + org.forstdb.RestoreOptions\ + org.forstdb.RocksCallbackObject\ + org.forstdb.RocksDB\ + org.forstdb.RocksEnv\ + org.forstdb.RocksIterator\ + org.forstdb.RocksMemEnv\ + org.forstdb.SkipListMemTableConfig\ + org.forstdb.Slice\ + org.forstdb.SstFileManager\ + org.forstdb.SstFileWriter\ + org.forstdb.SstFileReader\ + org.forstdb.SstFileReaderIterator\ + org.forstdb.SstPartitionerFactory\ + org.forstdb.SstPartitionerFixedPrefixFactory\ + org.forstdb.Statistics\ + org.forstdb.ThreadStatus\ + org.forstdb.TimedEnv\ + org.forstdb.Transaction\ + org.forstdb.TransactionDB\ + org.forstdb.TransactionDBOptions\ + org.forstdb.TransactionOptions\ + org.forstdb.TransactionLogIterator\ + org.forstdb.TtlDB\ + org.forstdb.VectorMemTableConfig\ + org.forstdb.Snapshot\ + org.forstdb.StringAppendOperator\ + org.forstdb.UInt64AddOperator\ + org.forstdb.WriteBatch\ + org.forstdb.WriteBatch.Handler\ + org.forstdb.WriteOptions\ + org.forstdb.WriteBatchWithIndex\ + org.forstdb.WriteBufferManager\ + org.forstdb.WBWIRocksIterator NATIVE_JAVA_TEST_CLASSES = \ - org.rocksdb.RocksDBExceptionTest\ - org.rocksdb.test.TestableEventListener\ - org.rocksdb.NativeComparatorWrapperTest.NativeStringComparatorWrapper\ - org.rocksdb.WriteBatchTest\ - org.rocksdb.WriteBatchTestInternalHelper + org.forstdb.RocksDBExceptionTest\ + org.forstdb.test.TestableEventListener\ + org.forstdb.NativeComparatorWrapperTest.NativeStringComparatorWrapper\ + org.forstdb.WriteBatchTest\ + org.forstdb.WriteBatchTestInternalHelper ROCKSDB_MAJOR = $(shell grep -E "ROCKSDB_MAJOR.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3) ROCKSDB_MINOR = $(shell grep -E "ROCKSDB_MINOR.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3) @@ -106,102 +108,109 @@ ARCH := $(shell getconf LONG_BIT) SHA256_CMD ?= sha256sum JAVA_TESTS = \ - org.rocksdb.BackupEngineOptionsTest\ - org.rocksdb.BackupEngineTest\ - org.rocksdb.BlobOptionsTest\ - org.rocksdb.BlockBasedTableConfigTest\ - org.rocksdb.BuiltinComparatorTest\ - org.rocksdb.ByteBufferUnsupportedOperationTest\ - org.rocksdb.BytewiseComparatorRegressionTest\ - org.rocksdb.util.BytewiseComparatorTest\ - org.rocksdb.util.BytewiseComparatorIntTest\ - org.rocksdb.CheckPointTest\ - org.rocksdb.ClockCacheTest\ - org.rocksdb.ColumnFamilyOptionsTest\ - org.rocksdb.ColumnFamilyTest\ - org.rocksdb.CompactionFilterFactoryTest\ - org.rocksdb.CompactionJobInfoTest\ - org.rocksdb.CompactionJobStatsTest\ - org.rocksdb.CompactionOptionsTest\ - org.rocksdb.CompactionOptionsFIFOTest\ - org.rocksdb.CompactionOptionsUniversalTest\ - org.rocksdb.CompactionPriorityTest\ - org.rocksdb.CompactionStopStyleTest\ - org.rocksdb.ComparatorOptionsTest\ - org.rocksdb.CompressionOptionsTest\ - org.rocksdb.CompressionTypesTest\ - org.rocksdb.DBOptionsTest\ - org.rocksdb.DirectSliceTest\ - org.rocksdb.util.EnvironmentTest\ - org.rocksdb.EnvOptionsTest\ - org.rocksdb.EventListenerTest\ - org.rocksdb.IngestExternalFileOptionsTest\ - org.rocksdb.util.IntComparatorTest\ - org.rocksdb.util.JNIComparatorTest\ - org.rocksdb.FilterTest\ - org.rocksdb.FlushTest\ - org.rocksdb.ImportColumnFamilyTest\ - org.rocksdb.InfoLogLevelTest\ - org.rocksdb.KeyMayExistTest\ - org.rocksdb.ConcurrentTaskLimiterTest\ - org.rocksdb.LoggerTest\ - org.rocksdb.LRUCacheTest\ - org.rocksdb.MemoryUtilTest\ - org.rocksdb.MemTableTest\ - org.rocksdb.MergeTest\ - org.rocksdb.MultiColumnRegressionTest \ - org.rocksdb.MultiGetManyKeysTest\ - org.rocksdb.MultiGetTest\ - org.rocksdb.MixedOptionsTest\ - org.rocksdb.MutableColumnFamilyOptionsTest\ - org.rocksdb.MutableDBOptionsTest\ - org.rocksdb.MutableOptionsGetSetTest \ - org.rocksdb.NativeComparatorWrapperTest\ - org.rocksdb.NativeLibraryLoaderTest\ - org.rocksdb.OptimisticTransactionTest\ - org.rocksdb.OptimisticTransactionDBTest\ - org.rocksdb.OptimisticTransactionOptionsTest\ - org.rocksdb.OptionsUtilTest\ - org.rocksdb.OptionsTest\ - org.rocksdb.PlainTableConfigTest\ - org.rocksdb.RateLimiterTest\ - org.rocksdb.ReadOnlyTest\ - org.rocksdb.ReadOptionsTest\ - org.rocksdb.util.ReverseBytewiseComparatorIntTest\ - org.rocksdb.RocksDBTest\ - org.rocksdb.RocksDBExceptionTest\ - org.rocksdb.DefaultEnvTest\ - org.rocksdb.RocksIteratorTest\ - org.rocksdb.RocksMemEnvTest\ - org.rocksdb.util.SizeUnitTest\ - org.rocksdb.SecondaryDBTest\ - org.rocksdb.SliceTest\ - org.rocksdb.SnapshotTest\ - org.rocksdb.SstFileManagerTest\ - org.rocksdb.SstFileWriterTest\ - org.rocksdb.SstFileReaderTest\ - org.rocksdb.SstPartitionerTest\ - org.rocksdb.TableFilterTest\ - org.rocksdb.TimedEnvTest\ - org.rocksdb.TransactionTest\ - org.rocksdb.TransactionDBTest\ - org.rocksdb.TransactionOptionsTest\ - org.rocksdb.TransactionDBOptionsTest\ - org.rocksdb.TransactionLogIteratorTest\ - org.rocksdb.TtlDBTest\ - org.rocksdb.StatisticsTest\ - org.rocksdb.StatisticsCollectorTest\ - org.rocksdb.VerifyChecksumsTest\ - org.rocksdb.WalFilterTest\ - org.rocksdb.WALRecoveryModeTest\ - org.rocksdb.WriteBatchHandlerTest\ - org.rocksdb.WriteBatchTest\ - org.rocksdb.WriteBatchThreadedTest\ - org.rocksdb.WriteOptionsTest\ - org.rocksdb.WriteBatchWithIndexTest + org.forstdb.BackupEngineOptionsTest\ + org.forstdb.BackupEngineTest\ + org.forstdb.BlobOptionsTest\ + org.forstdb.BlockBasedTableConfigTest\ + org.forstdb.BuiltinComparatorTest\ + org.forstdb.ByteBufferUnsupportedOperationTest\ + org.forstdb.BytewiseComparatorRegressionTest\ + org.forstdb.util.BytewiseComparatorTest\ + org.forstdb.util.BytewiseComparatorIntTest\ + org.forstdb.CheckPointTest\ + org.forstdb.ClockCacheTest\ + org.forstdb.ColumnFamilyOptionsTest\ + org.forstdb.ColumnFamilyTest\ + org.forstdb.CompactionFilterFactoryTest\ + org.forstdb.CompactionJobInfoTest\ + org.forstdb.CompactionJobStatsTest\ + org.forstdb.CompactionOptionsTest\ + org.forstdb.CompactionOptionsFIFOTest\ + org.forstdb.CompactionOptionsUniversalTest\ + org.forstdb.CompactionPriorityTest\ + org.forstdb.CompactionStopStyleTest\ + org.forstdb.ComparatorOptionsTest\ + org.forstdb.CompressionOptionsTest\ + org.forstdb.CompressionTypesTest\ + org.forstdb.DBOptionsTest\ + org.forstdb.DirectSliceTest\ + org.forstdb.util.EnvironmentTest\ + org.forstdb.EnvOptionsTest\ + org.forstdb.EventListenerTest\ + org.forstdb.IngestExternalFileOptionsTest\ + org.forstdb.util.IntComparatorTest\ + org.forstdb.util.JNIComparatorTest\ + org.forstdb.FilterTest\ + org.forstdb.FlushTest\ + org.forstdb.ImportColumnFamilyTest\ + org.forstdb.InfoLogLevelTest\ + org.forstdb.KeyExistsTest \ + org.forstdb.KeyMayExistTest\ + org.forstdb.ConcurrentTaskLimiterTest\ + org.forstdb.LoggerTest\ + org.forstdb.LRUCacheTest\ + org.forstdb.MemoryUtilTest\ + org.forstdb.MemTableTest\ + org.forstdb.MergeCFVariantsTest\ + org.forstdb.MergeTest\ + org.forstdb.MergeVariantsTest\ + org.forstdb.MultiColumnRegressionTest \ + org.forstdb.MultiGetManyKeysTest\ + org.forstdb.MultiGetTest\ + org.forstdb.MixedOptionsTest\ + org.forstdb.MutableColumnFamilyOptionsTest\ + org.forstdb.MutableDBOptionsTest\ + org.forstdb.MutableOptionsGetSetTest \ + org.forstdb.NativeComparatorWrapperTest\ + org.forstdb.NativeLibraryLoaderTest\ + org.forstdb.OptimisticTransactionTest\ + org.forstdb.OptimisticTransactionDBTest\ + org.forstdb.OptimisticTransactionOptionsTest\ + org.forstdb.OptionsUtilTest\ + org.forstdb.OptionsTest\ + org.forstdb.PerfLevelTest \ + org.forstdb.PerfContextTest \ + org.forstdb.PutCFVariantsTest\ + org.forstdb.PutVariantsTest\ + org.forstdb.PlainTableConfigTest\ + org.forstdb.RateLimiterTest\ + org.forstdb.ReadOnlyTest\ + org.forstdb.ReadOptionsTest\ + org.forstdb.util.ReverseBytewiseComparatorIntTest\ + org.forstdb.RocksDBTest\ + org.forstdb.RocksDBExceptionTest\ + org.forstdb.DefaultEnvTest\ + org.forstdb.RocksIteratorTest\ + org.forstdb.RocksMemEnvTest\ + org.forstdb.util.SizeUnitTest\ + org.forstdb.SecondaryDBTest\ + org.forstdb.SliceTest\ + org.forstdb.SnapshotTest\ + org.forstdb.SstFileManagerTest\ + org.forstdb.SstFileWriterTest\ + org.forstdb.SstFileReaderTest\ + org.forstdb.SstPartitionerTest\ + org.forstdb.TableFilterTest\ + org.forstdb.TimedEnvTest\ + org.forstdb.TransactionTest\ + org.forstdb.TransactionDBTest\ + org.forstdb.TransactionOptionsTest\ + org.forstdb.TransactionDBOptionsTest\ + org.forstdb.TransactionLogIteratorTest\ + org.forstdb.TtlDBTest\ + org.forstdb.StatisticsTest\ + org.forstdb.StatisticsCollectorTest\ + org.forstdb.VerifyChecksumsTest\ + org.forstdb.WalFilterTest\ + org.forstdb.WALRecoveryModeTest\ + org.forstdb.WriteBatchHandlerTest\ + org.forstdb.WriteBatchTest\ + org.forstdb.WriteBatchThreadedTest\ + org.forstdb.WriteOptionsTest\ + org.forstdb.WriteBatchWithIndexTest FLINK_TESTS = \ - org.rocksdb.flink.FlinkEnvTest + org.forstdb.flink.FlinkEnvTest MAIN_SRC = src/main/java TEST_SRC = src/test/java @@ -268,6 +277,8 @@ JAVADOC_CMD := javadoc endif endif +MAVEN_CMD := mvn + # Look for the Java version (1.6->6, 1.7->7, 1.8->8, 11.0->11, 13.0->13, 15.0->15 etc..) JAVAC_VERSION := $(shell $(JAVAC_CMD) -version 2>&1) JAVAC_MAJOR_VERSION := $(word 2,$(subst ., ,$(JAVAC_VERSION))) @@ -291,11 +302,11 @@ include $(ROCKSDB_PLUGIN_MKS) # Add paths to Java sources in plugins ROCKSDB_PLUGIN_JAVA_ROOTS = $(foreach plugin, $(ROCKSDB_PLUGINS), $(PLUGIN_PATH)/$(plugin)/java) -PLUGIN_SOURCES = $(foreach root, $(ROCKSDB_PLUGIN_JAVA_ROOTS), $(foreach pkg, org/rocksdb/util org/rocksdb, $(root)/$(MAIN_SRC)/$(pkg)/*.java)) -CORE_SOURCES = $(foreach pkg, org/rocksdb/util org/rocksdb, $(MAIN_SRC)/$(pkg)/*.java) +PLUGIN_SOURCES = $(foreach root, $(ROCKSDB_PLUGIN_JAVA_ROOTS), $(foreach pkg, org/forstdb/util org/forstdb, $(root)/$(MAIN_SRC)/$(pkg)/*.java)) +CORE_SOURCES = $(foreach pkg, org/forstdb/util org/forstdb, $(MAIN_SRC)/$(pkg)/*.java) SOURCES = $(wildcard $(CORE_SOURCES) $(PLUGIN_SOURCES)) -PLUGIN_TEST_SOURCES = $(foreach root, $(ROCKSDB_PLUGIN_JAVA_ROOTS), $(foreach pkg, org/rocksdb/test org/rocksdb/util org/rocksdb, $(root)/$(TEST_SRC)/$(pkg)/*.java)) -CORE_TEST_SOURCES = $(foreach pkg, org/rocksdb/test org/rocksdb/util org/rocksdb/flink org/rocksdb, $(TEST_SRC)/$(pkg)/*.java) +PLUGIN_TEST_SOURCES = $(foreach root, $(ROCKSDB_PLUGIN_JAVA_ROOTS), $(foreach pkg, org/forstdb/test org/forstdb/util org/forstdb, $(root)/$(TEST_SRC)/$(pkg)/*.java)) +CORE_TEST_SOURCES = $(foreach pkg, org/forstdb/test org/forstdb/util org/forstdb/flink org/forstdb, $(TEST_SRC)/$(pkg)/*.java) TEST_SOURCES = $(wildcard $(CORE_TEST_SOURCES) $(PLUGIN_TEST_SOURCES)) MOCK_FLINK_TEST_SOURCES = $(foreach pkg, org/apache/flink/core/fs org/apache/flink/state/forst/fs, flinktestmock/src/main/java/$(pkg)/*.java) @@ -346,32 +357,32 @@ java: java-version sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java - $(AM_V_at)@rm -rf /tmp/rocksdbjni - $(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni - $(AM_V_at)@rm -rf /tmp/rocksdbjni - $(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found + $(AM_V_at)@rm -rf /tmp/forstjni + $(AM_V_at)@rm -rf /tmp/forstjni_not_found + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/forstjni + $(AM_V_at)@rm -rf /tmp/forstjni + $(AM_V_at)@rm -rf /tmp/forstjni_not_found column_family_sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java - $(AM_V_at)@rm -rf /tmp/rocksdbjni - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni - $(AM_V_at)@rm -rf /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/forstjni + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/forstjni + $(AM_V_at)@rm -rf /tmp/forstjni transaction_sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) $(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/TransactionSample.java - $(AM_V_at)@rm -rf /tmp/rocksdbjni - $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/rocksdbjni - $(AM_V_at)@rm -rf /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/forstjni + $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/forstjni + $(AM_V_at)@rm -rf /tmp/forstjni optimistic_transaction_sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) $(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/OptimisticTransactionSample.java - $(AM_V_at)@rm -rf /tmp/rocksdbjni - $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/rocksdbjni - $(AM_V_at)@rm -rf /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/forstjni + $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/forstjni + $(AM_V_at)@rm -rf /tmp/forstjni $(JAVA_TEST_LIBDIR): mkdir -p "$(JAVA_TEST_LIBDIR)" @@ -447,14 +458,18 @@ test: java mock_flink_fs java_test $(MAKE) run_test run_test: - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ALL_JAVA_TESTS) + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.forstdb.test.RocksJunitRunner $(ALL_JAVA_TESTS) + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.forstdb.test.RocksJunitRunner org.forstdb.StatisticsTest run_plugin_test: - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS) + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.forstdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS) db_bench: java $(AM_V_GEN)mkdir -p $(BENCHMARK_MAIN_CLASSES) - $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java + $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/forstdb/benchmark/*.java + +pmd: + $(MAVEN_CMD) pmd:pmd pmd:cpd pmd:check mock_flink_fs: $(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_TESTCLASSPATH) -h $(NATIVE_INCLUDE) -d $(TEST_CLASSES) \ @@ -464,4 +479,4 @@ flink_test: java java_test mock_flink_fs $(MAKE) run_flink_test run_flink_test: - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(FLINK_TESTS) + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.forstdb.test.RocksJunitRunner $(FLINK_TESTS) diff --git a/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java b/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java index 070f0fe75..4b8372f51 100644 --- a/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java +++ b/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java @@ -19,7 +19,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.rocksdb.benchmark; +package org.forstdb.benchmark; import java.io.IOException; import java.lang.Runnable; @@ -43,9 +43,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.rocksdb.*; -import org.rocksdb.RocksMemEnv; -import org.rocksdb.util.SizeUnit; +import org.forstdb.*; +import org.forstdb.RocksMemEnv; +import org.forstdb.util.SizeUnit; class Stats { int id_; diff --git a/java/crossbuild/build-linux-alpine.sh b/java/crossbuild/build-linux-alpine.sh index 561d34141..646f9bff9 100755 --- a/java/crossbuild/build-linux-alpine.sh +++ b/java/crossbuild/build-linux-alpine.sh @@ -66,5 +66,5 @@ cd /tmp &&\ cd /rocksdb make jclean clean PORTABLE=1 make -j8 rocksdbjavastatic -cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build -cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build +cp /rocksdb/java/target/libforstjni-* /rocksdb-build +cp /rocksdb/java/target/forstjni-* /rocksdb-build diff --git a/java/crossbuild/build-linux-centos.sh b/java/crossbuild/build-linux-centos.sh index 176e3456c..e00729246 100755 --- a/java/crossbuild/build-linux-centos.sh +++ b/java/crossbuild/build-linux-centos.sh @@ -34,5 +34,5 @@ export PATH=$JAVA_HOME:/usr/local/bin:$PATH cd /rocksdb scl enable devtoolset-2 'make clean-not-downloaded' scl enable devtoolset-2 'PORTABLE=1 make -j8 rocksdbjavastatic' -cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build -cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build +cp /rocksdb/java/target/libforstjni-* /rocksdb-build +cp /rocksdb/java/target/forstjni-* /rocksdb-build diff --git a/java/crossbuild/build-linux.sh b/java/crossbuild/build-linux.sh index 74178adb5..34caa57ea 100755 --- a/java/crossbuild/build-linux.sh +++ b/java/crossbuild/build-linux.sh @@ -9,7 +9,7 @@ export JAVA_HOME=$(echo /usr/lib/jvm/java-7-openjdk*) cd /rocksdb make jclean clean make -j 4 rocksdbjavastatic -cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build -cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build +cp /rocksdb/java/target/libforstjni-* /rocksdb-build +cp /rocksdb/java/target/forstjni-* /rocksdb-build sudo shutdown -h now diff --git a/java/crossbuild/build-win.bat b/java/crossbuild/build-win.bat index 2925ec19a..d0bea9f80 100644 --- a/java/crossbuild/build-win.bat +++ b/java/crossbuild/build-win.bat @@ -12,5 +12,5 @@ cmake -G "Visual Studio 15 Win64" -DWITH_JNI=1 .. cd .. -copy build\java\Release\rocksdbjni-shared.dll librocksdbjni-win64.dll -echo Result is in librocksdbjni-win64.dll \ No newline at end of file +copy build\java\Release\forstjni-shared.dll libforstjni-win64.dll +echo Result is in libforstjni-win64.dll \ No newline at end of file diff --git a/java/crossbuild/docker-build-linux-alpine.sh b/java/crossbuild/docker-build-linux-alpine.sh index e3e852efe..fddef0ff1 100755 --- a/java/crossbuild/docker-build-linux-alpine.sh +++ b/java/crossbuild/docker-build-linux-alpine.sh @@ -14,4 +14,4 @@ cd /rocksdb-local-build make clean-not-downloaded PORTABLE=1 make -j2 rocksdbjavastatic -cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar java/target/rocksdbjni-*-linux*.jar.sha1 /rocksdb-java-target +cp java/target/libforstjni-linux*.so java/target/forstjni-*-linux*.jar java/target/forstjni-*-linux*.jar.sha1 /rocksdb-java-target diff --git a/java/crossbuild/docker-build-linux-centos.sh b/java/crossbuild/docker-build-linux-centos.sh index 16581dec7..30ab8c39e 100755 --- a/java/crossbuild/docker-build-linux-centos.sh +++ b/java/crossbuild/docker-build-linux-centos.sh @@ -34,5 +34,5 @@ else PORTABLE=1 make -j2 rocksdbjavastatic fi -cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar java/target/rocksdbjni-*-linux*.jar.sha1 /rocksdb-java-target +cp java/target/libforstjni-linux*.so java/target/forstjni-*-linux*.jar java/target/forstjni-*-linux*.jar.sha1 /rocksdb-java-target diff --git a/java/rocksjni/backup_engine_options.cc b/java/forstjni/backup_engine_options.cc similarity index 77% rename from java/rocksjni/backup_engine_options.cc rename to java/forstjni/backup_engine_options.cc index 25bfb6720..589a711be 100644 --- a/java/rocksjni/backup_engine_options.cc +++ b/java/forstjni/backup_engine_options.cc @@ -14,20 +14,20 @@ #include #include -#include "include/org_rocksdb_BackupEngineOptions.h" +#include "include/org_forstdb_BackupEngineOptions.h" #include "rocksdb/utilities/backup_engine.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /////////////////////////////////////////////////////////////////////////// // BackupDBOptions /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: newBackupEngineOptions * Signature: (Ljava/lang/String;)J */ -jlong Java_org_rocksdb_BackupEngineOptions_newBackupEngineOptions( +jlong Java_org_forstdb_BackupEngineOptions_newBackupEngineOptions( JNIEnv* env, jclass /*jcls*/, jstring jpath) { const char* cpath = env->GetStringUTFChars(jpath, nullptr); if (cpath == nullptr) { @@ -40,11 +40,11 @@ jlong Java_org_rocksdb_BackupEngineOptions_newBackupEngineOptions( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: backupDir * Signature: (J)Ljava/lang/String; */ -jstring Java_org_rocksdb_BackupEngineOptions_backupDir(JNIEnv* env, +jstring Java_org_forstdb_BackupEngineOptions_backupDir(JNIEnv* env, jobject /*jopt*/, jlong jhandle) { auto* bopt = @@ -53,11 +53,11 @@ jstring Java_org_rocksdb_BackupEngineOptions_backupDir(JNIEnv* env, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setBackupEnv * Signature: (JJ)V */ -void Java_org_rocksdb_BackupEngineOptions_setBackupEnv( +void Java_org_forstdb_BackupEngineOptions_setBackupEnv( JNIEnv* /*env*/, jobject /*jopt*/, jlong jhandle, jlong jrocks_env_handle) { auto* bopt = reinterpret_cast(jhandle); @@ -67,11 +67,11 @@ void Java_org_rocksdb_BackupEngineOptions_setBackupEnv( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setShareTableFiles * Signature: (JZ)V */ -void Java_org_rocksdb_BackupEngineOptions_setShareTableFiles(JNIEnv* /*env*/, +void Java_org_forstdb_BackupEngineOptions_setShareTableFiles(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean flag) { @@ -81,11 +81,11 @@ void Java_org_rocksdb_BackupEngineOptions_setShareTableFiles(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: shareTableFiles * Signature: (J)Z */ -jboolean Java_org_rocksdb_BackupEngineOptions_shareTableFiles(JNIEnv* /*env*/, +jboolean Java_org_forstdb_BackupEngineOptions_shareTableFiles(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = @@ -94,11 +94,11 @@ jboolean Java_org_rocksdb_BackupEngineOptions_shareTableFiles(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setInfoLog * Signature: (JJ)V */ -void Java_org_rocksdb_BackupEngineOptions_setInfoLog(JNIEnv* /*env*/, +void Java_org_forstdb_BackupEngineOptions_setInfoLog(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong /*jlogger_handle*/) { @@ -111,11 +111,11 @@ void Java_org_rocksdb_BackupEngineOptions_setInfoLog(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setSync * Signature: (JZ)V */ -void Java_org_rocksdb_BackupEngineOptions_setSync(JNIEnv* /*env*/, +void Java_org_forstdb_BackupEngineOptions_setSync(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean flag) { @@ -125,11 +125,11 @@ void Java_org_rocksdb_BackupEngineOptions_setSync(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: sync * Signature: (J)Z */ -jboolean Java_org_rocksdb_BackupEngineOptions_sync(JNIEnv* /*env*/, +jboolean Java_org_forstdb_BackupEngineOptions_sync(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = @@ -138,11 +138,11 @@ jboolean Java_org_rocksdb_BackupEngineOptions_sync(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setDestroyOldData * Signature: (JZ)V */ -void Java_org_rocksdb_BackupEngineOptions_setDestroyOldData(JNIEnv* /*env*/, +void Java_org_forstdb_BackupEngineOptions_setDestroyOldData(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean flag) { @@ -152,11 +152,11 @@ void Java_org_rocksdb_BackupEngineOptions_setDestroyOldData(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: destroyOldData * Signature: (J)Z */ -jboolean Java_org_rocksdb_BackupEngineOptions_destroyOldData(JNIEnv* /*env*/, +jboolean Java_org_forstdb_BackupEngineOptions_destroyOldData(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = @@ -165,11 +165,11 @@ jboolean Java_org_rocksdb_BackupEngineOptions_destroyOldData(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setBackupLogFiles * Signature: (JZ)V */ -void Java_org_rocksdb_BackupEngineOptions_setBackupLogFiles(JNIEnv* /*env*/, +void Java_org_forstdb_BackupEngineOptions_setBackupLogFiles(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean flag) { @@ -179,11 +179,11 @@ void Java_org_rocksdb_BackupEngineOptions_setBackupLogFiles(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: backupLogFiles * Signature: (J)Z */ -jboolean Java_org_rocksdb_BackupEngineOptions_backupLogFiles(JNIEnv* /*env*/, +jboolean Java_org_forstdb_BackupEngineOptions_backupLogFiles(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = @@ -192,11 +192,11 @@ jboolean Java_org_rocksdb_BackupEngineOptions_backupLogFiles(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setBackupRateLimit * Signature: (JJ)V */ -void Java_org_rocksdb_BackupEngineOptions_setBackupRateLimit( +void Java_org_forstdb_BackupEngineOptions_setBackupRateLimit( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jbackup_rate_limit) { auto* bopt = @@ -205,11 +205,11 @@ void Java_org_rocksdb_BackupEngineOptions_setBackupRateLimit( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: backupRateLimit * Signature: (J)J */ -jlong Java_org_rocksdb_BackupEngineOptions_backupRateLimit(JNIEnv* /*env*/, +jlong Java_org_forstdb_BackupEngineOptions_backupRateLimit(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = @@ -218,11 +218,11 @@ jlong Java_org_rocksdb_BackupEngineOptions_backupRateLimit(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setBackupRateLimiter * Signature: (JJ)V */ -void Java_org_rocksdb_BackupEngineOptions_setBackupRateLimiter( +void Java_org_forstdb_BackupEngineOptions_setBackupRateLimiter( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jrate_limiter_handle) { auto* bopt = @@ -234,11 +234,11 @@ void Java_org_rocksdb_BackupEngineOptions_setBackupRateLimiter( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setRestoreRateLimit * Signature: (JJ)V */ -void Java_org_rocksdb_BackupEngineOptions_setRestoreRateLimit( +void Java_org_forstdb_BackupEngineOptions_setRestoreRateLimit( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jrestore_rate_limit) { auto* bopt = @@ -247,11 +247,11 @@ void Java_org_rocksdb_BackupEngineOptions_setRestoreRateLimit( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: restoreRateLimit * Signature: (J)J */ -jlong Java_org_rocksdb_BackupEngineOptions_restoreRateLimit(JNIEnv* /*env*/, +jlong Java_org_forstdb_BackupEngineOptions_restoreRateLimit(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = @@ -260,11 +260,11 @@ jlong Java_org_rocksdb_BackupEngineOptions_restoreRateLimit(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setRestoreRateLimiter * Signature: (JJ)V */ -void Java_org_rocksdb_BackupEngineOptions_setRestoreRateLimiter( +void Java_org_forstdb_BackupEngineOptions_setRestoreRateLimiter( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jrate_limiter_handle) { auto* bopt = @@ -276,11 +276,11 @@ void Java_org_rocksdb_BackupEngineOptions_setRestoreRateLimiter( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setShareFilesWithChecksum * Signature: (JZ)V */ -void Java_org_rocksdb_BackupEngineOptions_setShareFilesWithChecksum( +void Java_org_forstdb_BackupEngineOptions_setShareFilesWithChecksum( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean flag) { auto* bopt = reinterpret_cast(jhandle); @@ -288,11 +288,11 @@ void Java_org_rocksdb_BackupEngineOptions_setShareFilesWithChecksum( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: shareFilesWithChecksum * Signature: (J)Z */ -jboolean Java_org_rocksdb_BackupEngineOptions_shareFilesWithChecksum( +jboolean Java_org_forstdb_BackupEngineOptions_shareFilesWithChecksum( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = reinterpret_cast(jhandle); @@ -300,11 +300,11 @@ jboolean Java_org_rocksdb_BackupEngineOptions_shareFilesWithChecksum( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setMaxBackgroundOperations * Signature: (JI)V */ -void Java_org_rocksdb_BackupEngineOptions_setMaxBackgroundOperations( +void Java_org_forstdb_BackupEngineOptions_setMaxBackgroundOperations( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jint max_background_operations) { auto* bopt = @@ -313,11 +313,11 @@ void Java_org_rocksdb_BackupEngineOptions_setMaxBackgroundOperations( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: maxBackgroundOperations * Signature: (J)I */ -jint Java_org_rocksdb_BackupEngineOptions_maxBackgroundOperations( +jint Java_org_forstdb_BackupEngineOptions_maxBackgroundOperations( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = reinterpret_cast(jhandle); @@ -325,11 +325,11 @@ jint Java_org_rocksdb_BackupEngineOptions_maxBackgroundOperations( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: setCallbackTriggerIntervalSize * Signature: (JJ)V */ -void Java_org_rocksdb_BackupEngineOptions_setCallbackTriggerIntervalSize( +void Java_org_forstdb_BackupEngineOptions_setCallbackTriggerIntervalSize( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jcallback_trigger_interval_size) { auto* bopt = @@ -339,11 +339,11 @@ void Java_org_rocksdb_BackupEngineOptions_setCallbackTriggerIntervalSize( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: callbackTriggerIntervalSize * Signature: (J)J */ -jlong Java_org_rocksdb_BackupEngineOptions_callbackTriggerIntervalSize( +jlong Java_org_forstdb_BackupEngineOptions_callbackTriggerIntervalSize( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* bopt = reinterpret_cast(jhandle); @@ -351,11 +351,11 @@ jlong Java_org_rocksdb_BackupEngineOptions_callbackTriggerIntervalSize( } /* - * Class: org_rocksdb_BackupEngineOptions + * Class: org_forstdb_BackupEngineOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_BackupEngineOptions_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_BackupEngineOptions_disposeInternal(JNIEnv* /*env*/, jobject /*jopt*/, jlong jhandle) { auto* bopt = diff --git a/java/rocksjni/backupenginejni.cc b/java/forstjni/backupenginejni.cc similarity index 86% rename from java/rocksjni/backupenginejni.cc rename to java/forstjni/backupenginejni.cc index 1ba7ea286..2a1876b4c 100644 --- a/java/rocksjni/backupenginejni.cc +++ b/java/forstjni/backupenginejni.cc @@ -10,17 +10,17 @@ #include -#include "include/org_rocksdb_BackupEngine.h" +#include "include/org_forstdb_BackupEngine.h" #include "rocksdb/utilities/backup_engine.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: open * Signature: (JJ)J */ -jlong Java_org_rocksdb_BackupEngine_open(JNIEnv* env, jclass /*jcls*/, +jlong Java_org_forstdb_BackupEngine_open(JNIEnv* env, jclass /*jcls*/, jlong env_handle, jlong backup_engine_options_handle) { auto* rocks_env = reinterpret_cast(env_handle); @@ -40,11 +40,11 @@ jlong Java_org_rocksdb_BackupEngine_open(JNIEnv* env, jclass /*jcls*/, } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: createNewBackup * Signature: (JJZ)V */ -void Java_org_rocksdb_BackupEngine_createNewBackup( +void Java_org_forstdb_BackupEngine_createNewBackup( JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jlong db_handle, jboolean jflush_before_backup) { auto* db = reinterpret_cast(db_handle); @@ -61,11 +61,11 @@ void Java_org_rocksdb_BackupEngine_createNewBackup( } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: createNewBackupWithMetadata * Signature: (JJLjava/lang/String;Z)V */ -void Java_org_rocksdb_BackupEngine_createNewBackupWithMetadata( +void Java_org_forstdb_BackupEngine_createNewBackupWithMetadata( JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jlong db_handle, jstring japp_metadata, jboolean jflush_before_backup) { auto* db = reinterpret_cast(db_handle); @@ -92,11 +92,11 @@ void Java_org_rocksdb_BackupEngine_createNewBackupWithMetadata( } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: getBackupInfo * Signature: (J)Ljava/util/List; */ -jobject Java_org_rocksdb_BackupEngine_getBackupInfo(JNIEnv* env, +jobject Java_org_forstdb_BackupEngine_getBackupInfo(JNIEnv* env, jobject /*jbe*/, jlong jbe_handle) { auto* backup_engine = @@ -107,11 +107,11 @@ jobject Java_org_rocksdb_BackupEngine_getBackupInfo(JNIEnv* env, } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: getCorruptedBackups * Signature: (J)[I */ -jintArray Java_org_rocksdb_BackupEngine_getCorruptedBackups(JNIEnv* env, +jintArray Java_org_forstdb_BackupEngine_getCorruptedBackups(JNIEnv* env, jobject /*jbe*/, jlong jbe_handle) { auto* backup_engine = @@ -135,11 +135,11 @@ jintArray Java_org_rocksdb_BackupEngine_getCorruptedBackups(JNIEnv* env, } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: garbageCollect * Signature: (J)V */ -void Java_org_rocksdb_BackupEngine_garbageCollect(JNIEnv* env, jobject /*jbe*/, +void Java_org_forstdb_BackupEngine_garbageCollect(JNIEnv* env, jobject /*jbe*/, jlong jbe_handle) { auto* backup_engine = reinterpret_cast(jbe_handle); @@ -153,11 +153,11 @@ void Java_org_rocksdb_BackupEngine_garbageCollect(JNIEnv* env, jobject /*jbe*/, } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: purgeOldBackups * Signature: (JI)V */ -void Java_org_rocksdb_BackupEngine_purgeOldBackups(JNIEnv* env, jobject /*jbe*/, +void Java_org_forstdb_BackupEngine_purgeOldBackups(JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jint jnum_backups_to_keep) { auto* backup_engine = @@ -173,11 +173,11 @@ void Java_org_rocksdb_BackupEngine_purgeOldBackups(JNIEnv* env, jobject /*jbe*/, } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: deleteBackup * Signature: (JI)V */ -void Java_org_rocksdb_BackupEngine_deleteBackup(JNIEnv* env, jobject /*jbe*/, +void Java_org_forstdb_BackupEngine_deleteBackup(JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jint jbackup_id) { auto* backup_engine = @@ -193,11 +193,11 @@ void Java_org_rocksdb_BackupEngine_deleteBackup(JNIEnv* env, jobject /*jbe*/, } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: restoreDbFromBackup * Signature: (JILjava/lang/String;Ljava/lang/String;J)V */ -void Java_org_rocksdb_BackupEngine_restoreDbFromBackup( +void Java_org_forstdb_BackupEngine_restoreDbFromBackup( JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jint jbackup_id, jstring jdb_dir, jstring jwal_dir, jlong jrestore_options_handle) { auto* backup_engine = @@ -230,11 +230,11 @@ void Java_org_rocksdb_BackupEngine_restoreDbFromBackup( } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: restoreDbFromLatestBackup * Signature: (JLjava/lang/String;Ljava/lang/String;J)V */ -void Java_org_rocksdb_BackupEngine_restoreDbFromLatestBackup( +void Java_org_forstdb_BackupEngine_restoreDbFromLatestBackup( JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jstring jdb_dir, jstring jwal_dir, jlong jrestore_options_handle) { auto* backup_engine = @@ -266,11 +266,11 @@ void Java_org_rocksdb_BackupEngine_restoreDbFromLatestBackup( } /* - * Class: org_rocksdb_BackupEngine + * Class: org_forstdb_BackupEngine * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_BackupEngine_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_BackupEngine_disposeInternal(JNIEnv* /*env*/, jobject /*jbe*/, jlong jbe_handle) { auto* be = reinterpret_cast(jbe_handle); diff --git a/java/rocksjni/cache.cc b/java/forstjni/cache.cc similarity index 78% rename from java/rocksjni/cache.cc rename to java/forstjni/cache.cc index 5ca1d5175..a1c863d35 100644 --- a/java/rocksjni/cache.cc +++ b/java/forstjni/cache.cc @@ -8,26 +8,26 @@ #include -#include "include/org_rocksdb_Cache.h" +#include "include/org_forstdb_Cache.h" #include "rocksdb/advanced_cache.h" /* - * Class: org_rocksdb_Cache + * Class: org_forstdb_Cache * Method: getUsage * Signature: (J)J */ -jlong Java_org_rocksdb_Cache_getUsage(JNIEnv*, jclass, jlong jhandle) { +jlong Java_org_forstdb_Cache_getUsage(JNIEnv*, jclass, jlong jhandle) { auto* sptr_cache = reinterpret_cast*>(jhandle); return static_cast(sptr_cache->get()->GetUsage()); } /* - * Class: org_rocksdb_Cache + * Class: org_forstdb_Cache * Method: getPinnedUsage * Signature: (J)J */ -jlong Java_org_rocksdb_Cache_getPinnedUsage(JNIEnv*, jclass, jlong jhandle) { +jlong Java_org_forstdb_Cache_getPinnedUsage(JNIEnv*, jclass, jlong jhandle) { auto* sptr_cache = reinterpret_cast*>(jhandle); return static_cast(sptr_cache->get()->GetPinnedUsage()); diff --git a/java/rocksjni/cassandra_compactionfilterjni.cc b/java/forstjni/cassandra_compactionfilterjni.cc similarity index 78% rename from java/rocksjni/cassandra_compactionfilterjni.cc rename to java/forstjni/cassandra_compactionfilterjni.cc index 25817aeca..805f31051 100644 --- a/java/rocksjni/cassandra_compactionfilterjni.cc +++ b/java/forstjni/cassandra_compactionfilterjni.cc @@ -5,16 +5,16 @@ #include -#include "include/org_rocksdb_CassandraCompactionFilter.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "include/org_forstdb_CassandraCompactionFilter.h" +#include "forstjni/cplusplus_to_java_convert.h" #include "utilities/cassandra/cassandra_compaction_filter.h" /* - * Class: org_rocksdb_CassandraCompactionFilter + * Class: org_forstdb_CassandraCompactionFilter * Method: createNewCassandraCompactionFilter0 * Signature: (ZI)J */ -jlong Java_org_rocksdb_CassandraCompactionFilter_createNewCassandraCompactionFilter0( +jlong Java_org_forstdb_CassandraCompactionFilter_createNewCassandraCompactionFilter0( JNIEnv* /*env*/, jclass /*jcls*/, jboolean purge_ttl_on_expiration, jint gc_grace_period_in_seconds) { auto* compaction_filter = diff --git a/java/rocksjni/cassandra_value_operator.cc b/java/forstjni/cassandra_value_operator.cc similarity index 77% rename from java/rocksjni/cassandra_value_operator.cc rename to java/forstjni/cassandra_value_operator.cc index 6de28c1b1..46f4caae5 100644 --- a/java/rocksjni/cassandra_value_operator.cc +++ b/java/forstjni/cassandra_value_operator.cc @@ -10,7 +10,7 @@ #include #include -#include "include/org_rocksdb_CassandraValueMergeOperator.h" +#include "include/org_forstdb_CassandraValueMergeOperator.h" #include "rocksdb/db.h" #include "rocksdb/memtablerep.h" #include "rocksdb/merge_operator.h" @@ -18,16 +18,16 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/statistics.h" #include "rocksdb/table.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" #include "utilities/cassandra/merge_operator.h" /* - * Class: org_rocksdb_CassandraValueMergeOperator + * Class: org_forstdb_CassandraValueMergeOperator * Method: newSharedCassandraValueMergeOperator * Signature: (II)J */ -jlong Java_org_rocksdb_CassandraValueMergeOperator_newSharedCassandraValueMergeOperator( +jlong Java_org_forstdb_CassandraValueMergeOperator_newSharedCassandraValueMergeOperator( JNIEnv* /*env*/, jclass /*jclazz*/, jint gcGracePeriodInSeconds, jint operands_limit) { auto* op = new std::shared_ptr( @@ -37,11 +37,11 @@ jlong Java_org_rocksdb_CassandraValueMergeOperator_newSharedCassandraValueMergeO } /* - * Class: org_rocksdb_CassandraValueMergeOperator + * Class: org_forstdb_CassandraValueMergeOperator * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_CassandraValueMergeOperator_disposeInternal( +void Java_org_forstdb_CassandraValueMergeOperator_disposeInternal( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* op = reinterpret_cast*>( diff --git a/java/rocksjni/checkpoint.cc b/java/forstjni/checkpoint.cc similarity index 76% rename from java/rocksjni/checkpoint.cc rename to java/forstjni/checkpoint.cc index 92db8400c..7a2fd3b8d 100644 --- a/java/rocksjni/checkpoint.cc +++ b/java/forstjni/checkpoint.cc @@ -14,16 +14,16 @@ #include -#include "include/org_rocksdb_Checkpoint.h" +#include "include/org_forstdb_Checkpoint.h" #include "rocksdb/db.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_Checkpoint + * Class: org_forstdb_Checkpoint * Method: newCheckpoint * Signature: (J)J */ -jlong Java_org_rocksdb_Checkpoint_newCheckpoint(JNIEnv* /*env*/, +jlong Java_org_forstdb_Checkpoint_newCheckpoint(JNIEnv* /*env*/, jclass /*jclazz*/, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -33,11 +33,11 @@ jlong Java_org_rocksdb_Checkpoint_newCheckpoint(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Checkpoint + * Class: org_forstdb_Checkpoint * Method: dispose * Signature: (J)V */ -void Java_org_rocksdb_Checkpoint_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_Checkpoint_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* checkpoint = reinterpret_cast(jhandle); @@ -46,11 +46,11 @@ void Java_org_rocksdb_Checkpoint_disposeInternal(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Checkpoint + * Class: org_forstdb_Checkpoint * Method: createCheckpoint * Signature: (JLjava/lang/String;)V */ -void Java_org_rocksdb_Checkpoint_createCheckpoint(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_Checkpoint_createCheckpoint(JNIEnv* env, jobject /*jobj*/, jlong jcheckpoint_handle, jstring jcheckpoint_path) { const char* checkpoint_path = env->GetStringUTFChars(jcheckpoint_path, 0); @@ -71,17 +71,17 @@ void Java_org_rocksdb_Checkpoint_createCheckpoint(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Checkpoint + * Class: org_forstdb_Checkpoint * Method: exportColumnFamily - * Signature: (JJLjava/lang/String;)Lorg/rocksdb/ExportImportFilesMetaData; + * Signature: (JJLjava/lang/String;)Lorg/forstdb/ExportImportFilesMetaData; */ -jobject Java_org_rocksdb_Checkpoint_exportColumnFamily( +jlong Java_org_forstdb_Checkpoint_exportColumnFamily( JNIEnv* env, jobject /*jobj*/, jlong jcheckpoint_handle, jlong jcolumn_family_handle, jstring jexport_path) { const char* export_path = env->GetStringUTFChars(jexport_path, 0); if (export_path == nullptr) { // exception thrown: OutOfMemoryError - return nullptr; + return 0; } auto* checkpoint = @@ -102,11 +102,5 @@ jobject Java_org_rocksdb_Checkpoint_exportColumnFamily( ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); } - jobject jexport_import_files_meta_data = nullptr; - if (metadata != nullptr) { - jexport_import_files_meta_data = - ROCKSDB_NAMESPACE::ExportImportFilesMetaDataJni:: - fromCppExportImportFilesMetaData(env, metadata); - } - return jexport_import_files_meta_data; + return GET_CPLUSPLUS_POINTER(metadata); } diff --git a/java/rocksjni/clock_cache.cc b/java/forstjni/clock_cache.cc similarity index 81% rename from java/rocksjni/clock_cache.cc rename to java/forstjni/clock_cache.cc index e04991aa9..e5778d15e 100644 --- a/java/rocksjni/clock_cache.cc +++ b/java/forstjni/clock_cache.cc @@ -10,15 +10,15 @@ #include -#include "include/org_rocksdb_ClockCache.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "include/org_forstdb_ClockCache.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_ClockCache + * Class: org_forstdb_ClockCache * Method: newClockCache * Signature: (JIZ)J */ -jlong Java_org_rocksdb_ClockCache_newClockCache( +jlong Java_org_forstdb_ClockCache_newClockCache( JNIEnv* /*env*/, jclass /*jcls*/, jlong jcapacity, jint jnum_shard_bits, jboolean jstrict_capacity_limit) { auto* sptr_clock_cache = new std::shared_ptr( @@ -29,11 +29,11 @@ jlong Java_org_rocksdb_ClockCache_newClockCache( } /* - * Class: org_rocksdb_ClockCache + * Class: org_forstdb_ClockCache * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_ClockCache_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_ClockCache_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_clock_cache = diff --git a/java/rocksjni/columnfamilyhandle.cc b/java/forstjni/columnfamilyhandle.cc similarity index 78% rename from java/rocksjni/columnfamilyhandle.cc rename to java/forstjni/columnfamilyhandle.cc index 4140580f0..e548a1674 100644 --- a/java/rocksjni/columnfamilyhandle.cc +++ b/java/forstjni/columnfamilyhandle.cc @@ -10,15 +10,15 @@ #include #include -#include "include/org_rocksdb_ColumnFamilyHandle.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_ColumnFamilyHandle.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_ColumnFamilyHandle + * Class: org_forstdb_ColumnFamilyHandle * Method: getName * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_ColumnFamilyHandle_getName(JNIEnv* env, +jbyteArray Java_org_forstdb_ColumnFamilyHandle_getName(JNIEnv* env, jobject /*jobj*/, jlong jhandle) { auto* cfh = reinterpret_cast(jhandle); @@ -27,11 +27,11 @@ jbyteArray Java_org_rocksdb_ColumnFamilyHandle_getName(JNIEnv* env, } /* - * Class: org_rocksdb_ColumnFamilyHandle + * Class: org_forstdb_ColumnFamilyHandle * Method: getID * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyHandle_getID(JNIEnv* /*env*/, +jint Java_org_forstdb_ColumnFamilyHandle_getID(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* cfh = reinterpret_cast(jhandle); @@ -40,11 +40,11 @@ jint Java_org_rocksdb_ColumnFamilyHandle_getID(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_ColumnFamilyHandle + * Class: org_forstdb_ColumnFamilyHandle * Method: getDescriptor - * Signature: (J)Lorg/rocksdb/ColumnFamilyDescriptor; + * Signature: (J)Lorg/forstdb/ColumnFamilyDescriptor; */ -jobject Java_org_rocksdb_ColumnFamilyHandle_getDescriptor(JNIEnv* env, +jobject Java_org_forstdb_ColumnFamilyHandle_getDescriptor(JNIEnv* env, jobject /*jobj*/, jlong jhandle) { auto* cfh = reinterpret_cast(jhandle); @@ -59,11 +59,11 @@ jobject Java_org_rocksdb_ColumnFamilyHandle_getDescriptor(JNIEnv* env, } /* - * Class: org_rocksdb_ColumnFamilyHandle + * Class: org_forstdb_ColumnFamilyHandle * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_ColumnFamilyHandle_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_ColumnFamilyHandle_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* cfh = reinterpret_cast(jhandle); diff --git a/java/rocksjni/compact_range_options.cc b/java/forstjni/compact_range_options.cc similarity index 69% rename from java/rocksjni/compact_range_options.cc rename to java/forstjni/compact_range_options.cc index d07263ab6..6f3e1c84e 100644 --- a/java/rocksjni/compact_range_options.cc +++ b/java/forstjni/compact_range_options.cc @@ -8,10 +8,10 @@ #include -#include "include/org_rocksdb_CompactRangeOptions.h" +#include "include/org_forstdb_CompactRangeOptions.h" #include "rocksdb/options.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" #include "util/coding.h" /** @@ -23,7 +23,7 @@ * maintain the lifetime of these parameters (`full_history_ts_low`, `canceled`) * by including their values in this class. */ -class Java_org_rocksdb_CompactRangeOptions { +class Java_org_forstdb_CompactRangeOptions { public: ROCKSDB_NAMESPACE::CompactRangeOptions compactRangeOptions; @@ -64,229 +64,229 @@ class Java_org_rocksdb_CompactRangeOptions { }; /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: newCompactRangeOptions * Signature: ()J */ -jlong Java_org_rocksdb_CompactRangeOptions_newCompactRangeOptions( +jlong Java_org_forstdb_CompactRangeOptions_newCompactRangeOptions( JNIEnv* /*env*/, jclass /*jclazz*/) { - auto* options = new Java_org_rocksdb_CompactRangeOptions(); + auto* options = new Java_org_forstdb_CompactRangeOptions(); return GET_CPLUSPLUS_POINTER(&options->compactRangeOptions); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: exclusiveManualCompaction * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompactRangeOptions_exclusiveManualCompaction( +jboolean Java_org_forstdb_CompactRangeOptions_exclusiveManualCompaction( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); return static_cast( options->compactRangeOptions.exclusive_manual_compaction); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setExclusiveManualCompaction * Signature: (JZ)V */ -void Java_org_rocksdb_CompactRangeOptions_setExclusiveManualCompaction( +void Java_org_forstdb_CompactRangeOptions_setExclusiveManualCompaction( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean exclusive_manual_compaction) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->compactRangeOptions.exclusive_manual_compaction = static_cast(exclusive_manual_compaction); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: bottommostLevelCompaction * Signature: (J)I */ -jint Java_org_rocksdb_CompactRangeOptions_bottommostLevelCompaction( +jint Java_org_forstdb_CompactRangeOptions_bottommostLevelCompaction( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::BottommostLevelCompactionJni:: toJavaBottommostLevelCompaction( options->compactRangeOptions.bottommost_level_compaction); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setBottommostLevelCompaction * Signature: (JI)V */ -void Java_org_rocksdb_CompactRangeOptions_setBottommostLevelCompaction( +void Java_org_forstdb_CompactRangeOptions_setBottommostLevelCompaction( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jint bottommost_level_compaction) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->compactRangeOptions.bottommost_level_compaction = ROCKSDB_NAMESPACE::BottommostLevelCompactionJni:: toCppBottommostLevelCompaction(bottommost_level_compaction); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: changeLevel * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompactRangeOptions_changeLevel(JNIEnv* /*env*/, +jboolean Java_org_forstdb_CompactRangeOptions_changeLevel(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); return static_cast(options->compactRangeOptions.change_level); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setChangeLevel * Signature: (JZ)V */ -void Java_org_rocksdb_CompactRangeOptions_setChangeLevel( +void Java_org_forstdb_CompactRangeOptions_setChangeLevel( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean change_level) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->compactRangeOptions.change_level = static_cast(change_level); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: targetLevel * Signature: (J)I */ -jint Java_org_rocksdb_CompactRangeOptions_targetLevel(JNIEnv* /*env*/, +jint Java_org_forstdb_CompactRangeOptions_targetLevel(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); return static_cast(options->compactRangeOptions.target_level); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setTargetLevel * Signature: (JI)V */ -void Java_org_rocksdb_CompactRangeOptions_setTargetLevel(JNIEnv* /*env*/, +void Java_org_forstdb_CompactRangeOptions_setTargetLevel(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jint target_level) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->compactRangeOptions.target_level = static_cast(target_level); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: targetPathId * Signature: (J)I */ -jint Java_org_rocksdb_CompactRangeOptions_targetPathId(JNIEnv* /*env*/, +jint Java_org_forstdb_CompactRangeOptions_targetPathId(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); return static_cast(options->compactRangeOptions.target_path_id); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setTargetPathId * Signature: (JI)V */ -void Java_org_rocksdb_CompactRangeOptions_setTargetPathId(JNIEnv* /*env*/, +void Java_org_forstdb_CompactRangeOptions_setTargetPathId(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jint target_path_id) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->compactRangeOptions.target_path_id = static_cast(target_path_id); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: allowWriteStall * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompactRangeOptions_allowWriteStall(JNIEnv* /*env*/, +jboolean Java_org_forstdb_CompactRangeOptions_allowWriteStall(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); return static_cast(options->compactRangeOptions.allow_write_stall); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setAllowWriteStall * Signature: (JZ)V */ -void Java_org_rocksdb_CompactRangeOptions_setAllowWriteStall( +void Java_org_forstdb_CompactRangeOptions_setAllowWriteStall( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean allow_write_stall) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->compactRangeOptions.allow_write_stall = static_cast(allow_write_stall); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: maxSubcompactions * Signature: (J)I */ -jint Java_org_rocksdb_CompactRangeOptions_maxSubcompactions(JNIEnv* /*env*/, +jint Java_org_forstdb_CompactRangeOptions_maxSubcompactions(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); return static_cast(options->compactRangeOptions.max_subcompactions); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setMaxSubcompactions * Signature: (JI)V */ -void Java_org_rocksdb_CompactRangeOptions_setMaxSubcompactions( +void Java_org_forstdb_CompactRangeOptions_setMaxSubcompactions( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jint max_subcompactions) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->compactRangeOptions.max_subcompactions = static_cast(max_subcompactions); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setFullHistoryTSLow * Signature: (JJJ)V */ -void Java_org_rocksdb_CompactRangeOptions_setFullHistoryTSLow(JNIEnv*, jobject, +void Java_org_forstdb_CompactRangeOptions_setFullHistoryTSLow(JNIEnv*, jobject, jlong jhandle, jlong start, jlong range) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->set_full_history_ts_low(start, range); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: fullHistoryTSLow - * Signature: (J)Lorg/rocksdb/CompactRangeOptions/Timestamp; + * Signature: (J)Lorg/forstdb/CompactRangeOptions/Timestamp; */ -jobject Java_org_rocksdb_CompactRangeOptions_fullHistoryTSLow(JNIEnv* env, +jobject Java_org_forstdb_CompactRangeOptions_fullHistoryTSLow(JNIEnv* env, jobject, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); uint64_t start; uint64_t range; jobject result = nullptr; @@ -300,39 +300,39 @@ jobject Java_org_rocksdb_CompactRangeOptions_fullHistoryTSLow(JNIEnv* env, } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: setCanceled * Signature: (JZ)V */ -void Java_org_rocksdb_CompactRangeOptions_setCanceled(JNIEnv*, jobject, +void Java_org_forstdb_CompactRangeOptions_setCanceled(JNIEnv*, jobject, jlong jhandle, jboolean jcanceled) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); options->set_canceled(jcanceled); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: canceled * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompactRangeOptions_canceled(JNIEnv*, jobject, +jboolean Java_org_forstdb_CompactRangeOptions_canceled(JNIEnv*, jobject, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); return options->get_canceled(); } /* - * Class: org_rocksdb_CompactRangeOptions + * Class: org_forstdb_CompactRangeOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_CompactRangeOptions_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_CompactRangeOptions_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* options = - reinterpret_cast(jhandle); + reinterpret_cast(jhandle); delete options; } diff --git a/java/rocksjni/compaction_filter.cc b/java/forstjni/compaction_filter.cc similarity index 77% rename from java/rocksjni/compaction_filter.cc rename to java/forstjni/compaction_filter.cc index ea04996ac..739447f96 100644 --- a/java/rocksjni/compaction_filter.cc +++ b/java/forstjni/compaction_filter.cc @@ -10,16 +10,16 @@ #include -#include "include/org_rocksdb_AbstractCompactionFilter.h" +#include "include/org_forstdb_AbstractCompactionFilter.h" -// +// /* - * Class: org_rocksdb_AbstractCompactionFilter + * Class: org_forstdb_AbstractCompactionFilter * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_AbstractCompactionFilter_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_AbstractCompactionFilter_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { auto* cf = reinterpret_cast(handle); diff --git a/java/rocksjni/compaction_filter_factory.cc b/java/forstjni/compaction_filter_factory.cc similarity index 71% rename from java/rocksjni/compaction_filter_factory.cc rename to java/forstjni/compaction_filter_factory.cc index 16fbdbbdd..5f68420c3 100644 --- a/java/rocksjni/compaction_filter_factory.cc +++ b/java/forstjni/compaction_filter_factory.cc @@ -10,16 +10,16 @@ #include -#include "include/org_rocksdb_AbstractCompactionFilterFactory.h" -#include "rocksjni/compaction_filter_factory_jnicallback.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "include/org_forstdb_AbstractCompactionFilterFactory.h" +#include "forstjni/compaction_filter_factory_jnicallback.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_AbstractCompactionFilterFactory + * Class: org_forstdb_AbstractCompactionFilterFactory * Method: createNewCompactionFilterFactory0 * Signature: ()J */ -jlong Java_org_rocksdb_AbstractCompactionFilterFactory_createNewCompactionFilterFactory0( +jlong Java_org_forstdb_AbstractCompactionFilterFactory_createNewCompactionFilterFactory0( JNIEnv* env, jobject jobj) { auto* cff = new ROCKSDB_NAMESPACE::CompactionFilterFactoryJniCallback(env, jobj); @@ -29,11 +29,11 @@ jlong Java_org_rocksdb_AbstractCompactionFilterFactory_createNewCompactionFilter } /* - * Class: org_rocksdb_AbstractCompactionFilterFactory + * Class: org_forstdb_AbstractCompactionFilterFactory * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_AbstractCompactionFilterFactory_disposeInternal( +void Java_org_forstdb_AbstractCompactionFilterFactory_disposeInternal( JNIEnv*, jobject, jlong jhandle) { auto* ptr_sptr_cff = reinterpret_cast< std::shared_ptr*>( diff --git a/java/rocksjni/compaction_filter_factory_jnicallback.cc b/java/forstjni/compaction_filter_factory_jnicallback.cc similarity index 96% rename from java/rocksjni/compaction_filter_factory_jnicallback.cc rename to java/forstjni/compaction_filter_factory_jnicallback.cc index 14285526f..ccf08eb0b 100644 --- a/java/rocksjni/compaction_filter_factory_jnicallback.cc +++ b/java/forstjni/compaction_filter_factory_jnicallback.cc @@ -6,9 +6,9 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::CompactionFilterFactory. -#include "rocksjni/compaction_filter_factory_jnicallback.h" +#include "forstjni/compaction_filter_factory_jnicallback.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { CompactionFilterFactoryJniCallback::CompactionFilterFactoryJniCallback( diff --git a/java/rocksjni/compaction_filter_factory_jnicallback.h b/java/forstjni/compaction_filter_factory_jnicallback.h similarity index 97% rename from java/rocksjni/compaction_filter_factory_jnicallback.h rename to java/forstjni/compaction_filter_factory_jnicallback.h index 2f26f8dbe..c8f1e718b 100644 --- a/java/rocksjni/compaction_filter_factory_jnicallback.h +++ b/java/forstjni/compaction_filter_factory_jnicallback.h @@ -14,7 +14,7 @@ #include #include "rocksdb/compaction_filter.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/compaction_job_info.cc b/java/forstjni/compaction_job_info.cc similarity index 79% rename from java/rocksjni/compaction_job_info.cc rename to java/forstjni/compaction_job_info.cc index fb292f59c..4097876ba 100644 --- a/java/rocksjni/compaction_job_info.cc +++ b/java/forstjni/compaction_job_info.cc @@ -8,27 +8,27 @@ #include -#include "include/org_rocksdb_CompactionJobInfo.h" +#include "include/org_forstdb_CompactionJobInfo.h" #include "rocksdb/listener.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: newCompactionJobInfo * Signature: ()J */ -jlong Java_org_rocksdb_CompactionJobInfo_newCompactionJobInfo(JNIEnv*, jclass) { +jlong Java_org_forstdb_CompactionJobInfo_newCompactionJobInfo(JNIEnv*, jclass) { auto* compact_job_info = new ROCKSDB_NAMESPACE::CompactionJobInfo(); return GET_CPLUSPLUS_POINTER(compact_job_info); } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_CompactionJobInfo_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_CompactionJobInfo_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -36,11 +36,11 @@ void Java_org_rocksdb_CompactionJobInfo_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: columnFamilyName * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_CompactionJobInfo_columnFamilyName(JNIEnv* env, +jbyteArray Java_org_forstdb_CompactionJobInfo_columnFamilyName(JNIEnv* env, jclass, jlong jhandle) { auto* compact_job_info = @@ -49,11 +49,11 @@ jbyteArray Java_org_rocksdb_CompactionJobInfo_columnFamilyName(JNIEnv* env, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: status - * Signature: (J)Lorg/rocksdb/Status; + * Signature: (J)Lorg/forstdb/Status; */ -jobject Java_org_rocksdb_CompactionJobInfo_status(JNIEnv* env, jclass, +jobject Java_org_forstdb_CompactionJobInfo_status(JNIEnv* env, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -61,11 +61,11 @@ jobject Java_org_rocksdb_CompactionJobInfo_status(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: threadId * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobInfo_threadId(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobInfo_threadId(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -73,22 +73,22 @@ jlong Java_org_rocksdb_CompactionJobInfo_threadId(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: jobId * Signature: (J)I */ -jint Java_org_rocksdb_CompactionJobInfo_jobId(JNIEnv*, jclass, jlong jhandle) { +jint Java_org_forstdb_CompactionJobInfo_jobId(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); return static_cast(compact_job_info->job_id); } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: baseInputLevel * Signature: (J)I */ -jint Java_org_rocksdb_CompactionJobInfo_baseInputLevel(JNIEnv*, jclass, +jint Java_org_forstdb_CompactionJobInfo_baseInputLevel(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -96,11 +96,11 @@ jint Java_org_rocksdb_CompactionJobInfo_baseInputLevel(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: outputLevel * Signature: (J)I */ -jint Java_org_rocksdb_CompactionJobInfo_outputLevel(JNIEnv*, jclass, +jint Java_org_forstdb_CompactionJobInfo_outputLevel(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -108,11 +108,11 @@ jint Java_org_rocksdb_CompactionJobInfo_outputLevel(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: inputFiles * Signature: (J)[Ljava/lang/String; */ -jobjectArray Java_org_rocksdb_CompactionJobInfo_inputFiles(JNIEnv* env, jclass, +jobjectArray Java_org_forstdb_CompactionJobInfo_inputFiles(JNIEnv* env, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -121,11 +121,11 @@ jobjectArray Java_org_rocksdb_CompactionJobInfo_inputFiles(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: outputFiles * Signature: (J)[Ljava/lang/String; */ -jobjectArray Java_org_rocksdb_CompactionJobInfo_outputFiles(JNIEnv* env, jclass, +jobjectArray Java_org_forstdb_CompactionJobInfo_outputFiles(JNIEnv* env, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -134,11 +134,11 @@ jobjectArray Java_org_rocksdb_CompactionJobInfo_outputFiles(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: tableProperties * Signature: (J)Ljava/util/Map; */ -jobject Java_org_rocksdb_CompactionJobInfo_tableProperties(JNIEnv* env, jclass, +jobject Java_org_forstdb_CompactionJobInfo_tableProperties(JNIEnv* env, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -191,11 +191,11 @@ jobject Java_org_rocksdb_CompactionJobInfo_tableProperties(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: compactionReason * Signature: (J)B */ -jbyte Java_org_rocksdb_CompactionJobInfo_compactionReason(JNIEnv*, jclass, +jbyte Java_org_forstdb_CompactionJobInfo_compactionReason(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -204,11 +204,11 @@ jbyte Java_org_rocksdb_CompactionJobInfo_compactionReason(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: compression * Signature: (J)B */ -jbyte Java_org_rocksdb_CompactionJobInfo_compression(JNIEnv*, jclass, +jbyte Java_org_forstdb_CompactionJobInfo_compression(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); @@ -217,11 +217,11 @@ jbyte Java_org_rocksdb_CompactionJobInfo_compression(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobInfo + * Class: org_forstdb_CompactionJobInfo * Method: stats * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobInfo_stats(JNIEnv*, jclass, jlong jhandle) { +jlong Java_org_forstdb_CompactionJobInfo_stats(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_info = reinterpret_cast(jhandle); auto* stats = new ROCKSDB_NAMESPACE::CompactionJobStats(); diff --git a/java/rocksjni/compaction_job_stats.cc b/java/forstjni/compaction_job_stats.cc similarity index 74% rename from java/rocksjni/compaction_job_stats.cc rename to java/forstjni/compaction_job_stats.cc index a2599c132..ca009e4af 100644 --- a/java/rocksjni/compaction_job_stats.cc +++ b/java/forstjni/compaction_job_stats.cc @@ -10,27 +10,27 @@ #include -#include "include/org_rocksdb_CompactionJobStats.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_CompactionJobStats.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: newCompactionJobStats * Signature: ()J */ -jlong Java_org_rocksdb_CompactionJobStats_newCompactionJobStats(JNIEnv*, +jlong Java_org_forstdb_CompactionJobStats_newCompactionJobStats(JNIEnv*, jclass) { auto* compact_job_stats = new ROCKSDB_NAMESPACE::CompactionJobStats(); return GET_CPLUSPLUS_POINTER(compact_job_stats); } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_CompactionJobStats_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_CompactionJobStats_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -38,22 +38,22 @@ void Java_org_rocksdb_CompactionJobStats_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: reset * Signature: (J)V */ -void Java_org_rocksdb_CompactionJobStats_reset(JNIEnv*, jclass, jlong jhandle) { +void Java_org_forstdb_CompactionJobStats_reset(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); compact_job_stats->Reset(); } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: add * Signature: (JJ)V */ -void Java_org_rocksdb_CompactionJobStats_add(JNIEnv*, jclass, jlong jhandle, +void Java_org_forstdb_CompactionJobStats_add(JNIEnv*, jclass, jlong jhandle, jlong jother_handle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -63,11 +63,11 @@ void Java_org_rocksdb_CompactionJobStats_add(JNIEnv*, jclass, jlong jhandle, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: elapsedMicros * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_elapsedMicros(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_elapsedMicros(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -75,11 +75,11 @@ jlong Java_org_rocksdb_CompactionJobStats_elapsedMicros(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numInputRecords * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numInputRecords(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_numInputRecords(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -87,11 +87,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numInputRecords(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numInputFiles * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numInputFiles(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_numInputFiles(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -99,11 +99,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numInputFiles(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numInputFilesAtOutputLevel * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numInputFilesAtOutputLevel( +jlong Java_org_forstdb_CompactionJobStats_numInputFilesAtOutputLevel( JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -111,11 +111,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numInputFilesAtOutputLevel( } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numOutputRecords * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numOutputRecords(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_numOutputRecords(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -123,11 +123,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numOutputRecords(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numOutputFiles * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numOutputFiles(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_numOutputFiles(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -135,11 +135,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numOutputFiles(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: isManualCompaction * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompactionJobStats_isManualCompaction(JNIEnv*, jclass, +jboolean Java_org_forstdb_CompactionJobStats_isManualCompaction(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -151,11 +151,11 @@ jboolean Java_org_rocksdb_CompactionJobStats_isManualCompaction(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: totalInputBytes * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_totalInputBytes(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_totalInputBytes(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -163,11 +163,11 @@ jlong Java_org_rocksdb_CompactionJobStats_totalInputBytes(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: totalOutputBytes * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_totalOutputBytes(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_totalOutputBytes(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -175,11 +175,11 @@ jlong Java_org_rocksdb_CompactionJobStats_totalOutputBytes(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numRecordsReplaced * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numRecordsReplaced(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_numRecordsReplaced(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -187,11 +187,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numRecordsReplaced(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: totalInputRawKeyBytes * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_totalInputRawKeyBytes(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_totalInputRawKeyBytes(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -199,11 +199,11 @@ jlong Java_org_rocksdb_CompactionJobStats_totalInputRawKeyBytes(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: totalInputRawValueBytes * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_totalInputRawValueBytes( +jlong Java_org_forstdb_CompactionJobStats_totalInputRawValueBytes( JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -211,11 +211,11 @@ jlong Java_org_rocksdb_CompactionJobStats_totalInputRawValueBytes( } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numInputDeletionRecords * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numInputDeletionRecords( +jlong Java_org_forstdb_CompactionJobStats_numInputDeletionRecords( JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -223,11 +223,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numInputDeletionRecords( } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numExpiredDeletionRecords * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numExpiredDeletionRecords( +jlong Java_org_forstdb_CompactionJobStats_numExpiredDeletionRecords( JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -235,11 +235,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numExpiredDeletionRecords( } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numCorruptKeys * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numCorruptKeys(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_numCorruptKeys(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -247,11 +247,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numCorruptKeys(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: fileWriteNanos * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_fileWriteNanos(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_fileWriteNanos(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -259,11 +259,11 @@ jlong Java_org_rocksdb_CompactionJobStats_fileWriteNanos(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: fileRangeSyncNanos * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_fileRangeSyncNanos(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_fileRangeSyncNanos(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -271,11 +271,11 @@ jlong Java_org_rocksdb_CompactionJobStats_fileRangeSyncNanos(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: fileFsyncNanos * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_fileFsyncNanos(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_fileFsyncNanos(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -283,11 +283,11 @@ jlong Java_org_rocksdb_CompactionJobStats_fileFsyncNanos(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: filePrepareWriteNanos * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_filePrepareWriteNanos(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_filePrepareWriteNanos(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -295,11 +295,11 @@ jlong Java_org_rocksdb_CompactionJobStats_filePrepareWriteNanos(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: smallestOutputKeyPrefix * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_CompactionJobStats_smallestOutputKeyPrefix( +jbyteArray Java_org_forstdb_CompactionJobStats_smallestOutputKeyPrefix( JNIEnv* env, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -308,11 +308,11 @@ jbyteArray Java_org_rocksdb_CompactionJobStats_smallestOutputKeyPrefix( } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: largestOutputKeyPrefix * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_CompactionJobStats_largestOutputKeyPrefix( +jbyteArray Java_org_forstdb_CompactionJobStats_largestOutputKeyPrefix( JNIEnv* env, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -321,11 +321,11 @@ jbyteArray Java_org_rocksdb_CompactionJobStats_largestOutputKeyPrefix( } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numSingleDelFallthru * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numSingleDelFallthru(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_numSingleDelFallthru(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); @@ -333,11 +333,11 @@ jlong Java_org_rocksdb_CompactionJobStats_numSingleDelFallthru(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionJobStats + * Class: org_forstdb_CompactionJobStats * Method: numSingleDelMismatch * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionJobStats_numSingleDelMismatch(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionJobStats_numSingleDelMismatch(JNIEnv*, jclass, jlong jhandle) { auto* compact_job_stats = reinterpret_cast(jhandle); diff --git a/java/rocksjni/compaction_options.cc b/java/forstjni/compaction_options.cc similarity index 74% rename from java/rocksjni/compaction_options.cc rename to java/forstjni/compaction_options.cc index bbbde0313..0cf3e92df 100644 --- a/java/rocksjni/compaction_options.cc +++ b/java/forstjni/compaction_options.cc @@ -8,27 +8,27 @@ #include -#include "include/org_rocksdb_CompactionOptions.h" +#include "include/org_forstdb_CompactionOptions.h" #include "rocksdb/options.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_CompactionOptions + * Class: org_forstdb_CompactionOptions * Method: newCompactionOptions * Signature: ()J */ -jlong Java_org_rocksdb_CompactionOptions_newCompactionOptions(JNIEnv*, jclass) { +jlong Java_org_forstdb_CompactionOptions_newCompactionOptions(JNIEnv*, jclass) { auto* compact_opts = new ROCKSDB_NAMESPACE::CompactionOptions(); return GET_CPLUSPLUS_POINTER(compact_opts); } /* - * Class: org_rocksdb_CompactionOptions + * Class: org_forstdb_CompactionOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_CompactionOptions_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_CompactionOptions_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* compact_opts = reinterpret_cast(jhandle); @@ -36,11 +36,11 @@ void Java_org_rocksdb_CompactionOptions_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompactionOptions + * Class: org_forstdb_CompactionOptions * Method: compression * Signature: (J)B */ -jbyte Java_org_rocksdb_CompactionOptions_compression(JNIEnv*, jclass, +jbyte Java_org_forstdb_CompactionOptions_compression(JNIEnv*, jclass, jlong jhandle) { auto* compact_opts = reinterpret_cast(jhandle); @@ -49,11 +49,11 @@ jbyte Java_org_rocksdb_CompactionOptions_compression(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionOptions + * Class: org_forstdb_CompactionOptions * Method: setCompression * Signature: (JB)V */ -void Java_org_rocksdb_CompactionOptions_setCompression( +void Java_org_forstdb_CompactionOptions_setCompression( JNIEnv*, jclass, jlong jhandle, jbyte jcompression_type_value) { auto* compact_opts = reinterpret_cast(jhandle); @@ -63,11 +63,11 @@ void Java_org_rocksdb_CompactionOptions_setCompression( } /* - * Class: org_rocksdb_CompactionOptions + * Class: org_forstdb_CompactionOptions * Method: outputFileSizeLimit * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionOptions_outputFileSizeLimit(JNIEnv*, jclass, +jlong Java_org_forstdb_CompactionOptions_outputFileSizeLimit(JNIEnv*, jclass, jlong jhandle) { auto* compact_opts = reinterpret_cast(jhandle); @@ -75,11 +75,11 @@ jlong Java_org_rocksdb_CompactionOptions_outputFileSizeLimit(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionOptions + * Class: org_forstdb_CompactionOptions * Method: setOutputFileSizeLimit * Signature: (JJ)V */ -void Java_org_rocksdb_CompactionOptions_setOutputFileSizeLimit( +void Java_org_forstdb_CompactionOptions_setOutputFileSizeLimit( JNIEnv*, jclass, jlong jhandle, jlong joutput_file_size_limit) { auto* compact_opts = reinterpret_cast(jhandle); @@ -88,11 +88,11 @@ void Java_org_rocksdb_CompactionOptions_setOutputFileSizeLimit( } /* - * Class: org_rocksdb_CompactionOptions + * Class: org_forstdb_CompactionOptions * Method: maxSubcompactions * Signature: (J)I */ -jint Java_org_rocksdb_CompactionOptions_maxSubcompactions(JNIEnv*, jclass, +jint Java_org_forstdb_CompactionOptions_maxSubcompactions(JNIEnv*, jclass, jlong jhandle) { auto* compact_opts = reinterpret_cast(jhandle); @@ -100,11 +100,11 @@ jint Java_org_rocksdb_CompactionOptions_maxSubcompactions(JNIEnv*, jclass, } /* - * Class: org_rocksdb_CompactionOptions + * Class: org_forstdb_CompactionOptions * Method: setMaxSubcompactions * Signature: (JI)V */ -void Java_org_rocksdb_CompactionOptions_setMaxSubcompactions( +void Java_org_forstdb_CompactionOptions_setMaxSubcompactions( JNIEnv*, jclass, jlong jhandle, jint jmax_subcompactions) { auto* compact_opts = reinterpret_cast(jhandle); diff --git a/java/rocksjni/compaction_options_fifo.cc b/java/forstjni/compaction_options_fifo.cc similarity index 73% rename from java/rocksjni/compaction_options_fifo.cc rename to java/forstjni/compaction_options_fifo.cc index f6a47fec5..3a4bf5c25 100644 --- a/java/rocksjni/compaction_options_fifo.cc +++ b/java/forstjni/compaction_options_fifo.cc @@ -8,27 +8,27 @@ #include -#include "include/org_rocksdb_CompactionOptionsFIFO.h" +#include "include/org_forstdb_CompactionOptionsFIFO.h" #include "rocksdb/advanced_options.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_CompactionOptionsFIFO + * Class: org_forstdb_CompactionOptionsFIFO * Method: newCompactionOptionsFIFO * Signature: ()J */ -jlong Java_org_rocksdb_CompactionOptionsFIFO_newCompactionOptionsFIFO(JNIEnv*, +jlong Java_org_forstdb_CompactionOptionsFIFO_newCompactionOptionsFIFO(JNIEnv*, jclass) { const auto* opt = new ROCKSDB_NAMESPACE::CompactionOptionsFIFO(); return GET_CPLUSPLUS_POINTER(opt); } /* - * Class: org_rocksdb_CompactionOptionsFIFO + * Class: org_forstdb_CompactionOptionsFIFO * Method: setMaxTableFilesSize * Signature: (JJ)V */ -void Java_org_rocksdb_CompactionOptionsFIFO_setMaxTableFilesSize( +void Java_org_forstdb_CompactionOptionsFIFO_setMaxTableFilesSize( JNIEnv*, jobject, jlong jhandle, jlong jmax_table_files_size) { auto* opt = reinterpret_cast(jhandle); @@ -36,11 +36,11 @@ void Java_org_rocksdb_CompactionOptionsFIFO_setMaxTableFilesSize( } /* - * Class: org_rocksdb_CompactionOptionsFIFO + * Class: org_forstdb_CompactionOptionsFIFO * Method: maxTableFilesSize * Signature: (J)J */ -jlong Java_org_rocksdb_CompactionOptionsFIFO_maxTableFilesSize(JNIEnv*, jobject, +jlong Java_org_forstdb_CompactionOptionsFIFO_maxTableFilesSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -48,11 +48,11 @@ jlong Java_org_rocksdb_CompactionOptionsFIFO_maxTableFilesSize(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompactionOptionsFIFO + * Class: org_forstdb_CompactionOptionsFIFO * Method: setAllowCompaction * Signature: (JZ)V */ -void Java_org_rocksdb_CompactionOptionsFIFO_setAllowCompaction( +void Java_org_forstdb_CompactionOptionsFIFO_setAllowCompaction( JNIEnv*, jobject, jlong jhandle, jboolean allow_compaction) { auto* opt = reinterpret_cast(jhandle); @@ -60,11 +60,11 @@ void Java_org_rocksdb_CompactionOptionsFIFO_setAllowCompaction( } /* - * Class: org_rocksdb_CompactionOptionsFIFO + * Class: org_forstdb_CompactionOptionsFIFO * Method: allowCompaction * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompactionOptionsFIFO_allowCompaction(JNIEnv*, +jboolean Java_org_forstdb_CompactionOptionsFIFO_allowCompaction(JNIEnv*, jobject, jlong jhandle) { auto* opt = @@ -73,11 +73,11 @@ jboolean Java_org_rocksdb_CompactionOptionsFIFO_allowCompaction(JNIEnv*, } /* - * Class: org_rocksdb_CompactionOptionsFIFO + * Class: org_forstdb_CompactionOptionsFIFO * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_CompactionOptionsFIFO_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_CompactionOptionsFIFO_disposeInternal(JNIEnv*, jobject, jlong jhandle) { delete reinterpret_cast(jhandle); } diff --git a/java/rocksjni/compaction_options_universal.cc b/java/forstjni/compaction_options_universal.cc similarity index 71% rename from java/rocksjni/compaction_options_universal.cc rename to java/forstjni/compaction_options_universal.cc index 9fc6f3158..c2fb1d6d4 100644 --- a/java/rocksjni/compaction_options_universal.cc +++ b/java/forstjni/compaction_options_universal.cc @@ -8,28 +8,28 @@ #include -#include "include/org_rocksdb_CompactionOptionsUniversal.h" +#include "include/org_forstdb_CompactionOptionsUniversal.h" #include "rocksdb/advanced_options.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: newCompactionOptionsUniversal * Signature: ()J */ -jlong Java_org_rocksdb_CompactionOptionsUniversal_newCompactionOptionsUniversal( +jlong Java_org_forstdb_CompactionOptionsUniversal_newCompactionOptionsUniversal( JNIEnv*, jclass) { const auto* opt = new ROCKSDB_NAMESPACE::CompactionOptionsUniversal(); return GET_CPLUSPLUS_POINTER(opt); } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: setSizeRatio * Signature: (JI)V */ -void Java_org_rocksdb_CompactionOptionsUniversal_setSizeRatio( +void Java_org_forstdb_CompactionOptionsUniversal_setSizeRatio( JNIEnv*, jobject, jlong jhandle, jint jsize_ratio) { auto* opt = reinterpret_cast(jhandle); @@ -37,11 +37,11 @@ void Java_org_rocksdb_CompactionOptionsUniversal_setSizeRatio( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: sizeRatio * Signature: (J)I */ -jint Java_org_rocksdb_CompactionOptionsUniversal_sizeRatio(JNIEnv*, jobject, +jint Java_org_forstdb_CompactionOptionsUniversal_sizeRatio(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -49,11 +49,11 @@ jint Java_org_rocksdb_CompactionOptionsUniversal_sizeRatio(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: setMinMergeWidth * Signature: (JI)V */ -void Java_org_rocksdb_CompactionOptionsUniversal_setMinMergeWidth( +void Java_org_forstdb_CompactionOptionsUniversal_setMinMergeWidth( JNIEnv*, jobject, jlong jhandle, jint jmin_merge_width) { auto* opt = reinterpret_cast(jhandle); @@ -61,11 +61,11 @@ void Java_org_rocksdb_CompactionOptionsUniversal_setMinMergeWidth( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: minMergeWidth * Signature: (J)I */ -jint Java_org_rocksdb_CompactionOptionsUniversal_minMergeWidth(JNIEnv*, jobject, +jint Java_org_forstdb_CompactionOptionsUniversal_minMergeWidth(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -73,11 +73,11 @@ jint Java_org_rocksdb_CompactionOptionsUniversal_minMergeWidth(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: setMaxMergeWidth * Signature: (JI)V */ -void Java_org_rocksdb_CompactionOptionsUniversal_setMaxMergeWidth( +void Java_org_forstdb_CompactionOptionsUniversal_setMaxMergeWidth( JNIEnv*, jobject, jlong jhandle, jint jmax_merge_width) { auto* opt = reinterpret_cast(jhandle); @@ -85,11 +85,11 @@ void Java_org_rocksdb_CompactionOptionsUniversal_setMaxMergeWidth( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: maxMergeWidth * Signature: (J)I */ -jint Java_org_rocksdb_CompactionOptionsUniversal_maxMergeWidth(JNIEnv*, jobject, +jint Java_org_forstdb_CompactionOptionsUniversal_maxMergeWidth(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -97,11 +97,11 @@ jint Java_org_rocksdb_CompactionOptionsUniversal_maxMergeWidth(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: setMaxSizeAmplificationPercent * Signature: (JI)V */ -void Java_org_rocksdb_CompactionOptionsUniversal_setMaxSizeAmplificationPercent( +void Java_org_forstdb_CompactionOptionsUniversal_setMaxSizeAmplificationPercent( JNIEnv*, jobject, jlong jhandle, jint jmax_size_amplification_percent) { auto* opt = reinterpret_cast(jhandle); @@ -110,11 +110,11 @@ void Java_org_rocksdb_CompactionOptionsUniversal_setMaxSizeAmplificationPercent( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: maxSizeAmplificationPercent * Signature: (J)I */ -jint Java_org_rocksdb_CompactionOptionsUniversal_maxSizeAmplificationPercent( +jint Java_org_forstdb_CompactionOptionsUniversal_maxSizeAmplificationPercent( JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -122,11 +122,11 @@ jint Java_org_rocksdb_CompactionOptionsUniversal_maxSizeAmplificationPercent( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: setCompressionSizePercent * Signature: (JI)V */ -void Java_org_rocksdb_CompactionOptionsUniversal_setCompressionSizePercent( +void Java_org_forstdb_CompactionOptionsUniversal_setCompressionSizePercent( JNIEnv*, jobject, jlong jhandle, jint jcompression_size_percent) { auto* opt = reinterpret_cast(jhandle); @@ -135,11 +135,11 @@ void Java_org_rocksdb_CompactionOptionsUniversal_setCompressionSizePercent( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: compressionSizePercent * Signature: (J)I */ -jint Java_org_rocksdb_CompactionOptionsUniversal_compressionSizePercent( +jint Java_org_forstdb_CompactionOptionsUniversal_compressionSizePercent( JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -147,11 +147,11 @@ jint Java_org_rocksdb_CompactionOptionsUniversal_compressionSizePercent( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: setStopStyle * Signature: (JB)V */ -void Java_org_rocksdb_CompactionOptionsUniversal_setStopStyle( +void Java_org_forstdb_CompactionOptionsUniversal_setStopStyle( JNIEnv*, jobject, jlong jhandle, jbyte jstop_style_value) { auto* opt = reinterpret_cast(jhandle); @@ -161,11 +161,11 @@ void Java_org_rocksdb_CompactionOptionsUniversal_setStopStyle( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: stopStyle * Signature: (J)B */ -jbyte Java_org_rocksdb_CompactionOptionsUniversal_stopStyle(JNIEnv*, jobject, +jbyte Java_org_forstdb_CompactionOptionsUniversal_stopStyle(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -174,11 +174,11 @@ jbyte Java_org_rocksdb_CompactionOptionsUniversal_stopStyle(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: setAllowTrivialMove * Signature: (JZ)V */ -void Java_org_rocksdb_CompactionOptionsUniversal_setAllowTrivialMove( +void Java_org_forstdb_CompactionOptionsUniversal_setAllowTrivialMove( JNIEnv*, jobject, jlong jhandle, jboolean jallow_trivial_move) { auto* opt = reinterpret_cast(jhandle); @@ -186,11 +186,11 @@ void Java_org_rocksdb_CompactionOptionsUniversal_setAllowTrivialMove( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: allowTrivialMove * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompactionOptionsUniversal_allowTrivialMove( +jboolean Java_org_forstdb_CompactionOptionsUniversal_allowTrivialMove( JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -198,11 +198,11 @@ jboolean Java_org_rocksdb_CompactionOptionsUniversal_allowTrivialMove( } /* - * Class: org_rocksdb_CompactionOptionsUniversal + * Class: org_forstdb_CompactionOptionsUniversal * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_CompactionOptionsUniversal_disposeInternal( +void Java_org_forstdb_CompactionOptionsUniversal_disposeInternal( JNIEnv*, jobject, jlong jhandle) { delete reinterpret_cast( jhandle); diff --git a/java/rocksjni/comparator.cc b/java/forstjni/comparator.cc similarity index 71% rename from java/rocksjni/comparator.cc rename to java/forstjni/comparator.cc index 11279c4ce..0e69990cf 100644 --- a/java/rocksjni/comparator.cc +++ b/java/forstjni/comparator.cc @@ -13,18 +13,18 @@ #include #include -#include "include/org_rocksdb_AbstractComparator.h" -#include "include/org_rocksdb_NativeComparatorWrapper.h" -#include "rocksjni/comparatorjnicallback.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_AbstractComparator.h" +#include "include/org_forstdb_NativeComparatorWrapper.h" +#include "forstjni/comparatorjnicallback.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_AbstractComparator + * Class: org_forstdb_AbstractComparator * Method: createNewComparator * Signature: (J)J */ -jlong Java_org_rocksdb_AbstractComparator_createNewComparator( +jlong Java_org_forstdb_AbstractComparator_createNewComparator( JNIEnv* env, jobject jcomparator, jlong copt_handle) { auto* copt = reinterpret_cast( @@ -35,11 +35,11 @@ jlong Java_org_rocksdb_AbstractComparator_createNewComparator( } /* - * Class: org_rocksdb_AbstractComparator + * Class: org_forstdb_AbstractComparator * Method: usingDirectBuffers * Signature: (J)Z */ -jboolean Java_org_rocksdb_AbstractComparator_usingDirectBuffers(JNIEnv*, +jboolean Java_org_forstdb_AbstractComparator_usingDirectBuffers(JNIEnv*, jobject, jlong jhandle) { auto* c = @@ -48,11 +48,11 @@ jboolean Java_org_rocksdb_AbstractComparator_usingDirectBuffers(JNIEnv*, } /* - * Class: org_rocksdb_NativeComparatorWrapper + * Class: org_forstdb_NativeComparatorWrapper * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_NativeComparatorWrapper_disposeInternal( +void Java_org_forstdb_NativeComparatorWrapper_disposeInternal( JNIEnv* /*env*/, jobject /*jobj*/, jlong jcomparator_handle) { auto* comparator = reinterpret_cast(jcomparator_handle); diff --git a/java/rocksjni/comparatorjnicallback.cc b/java/forstjni/comparatorjnicallback.cc similarity index 99% rename from java/rocksjni/comparatorjnicallback.cc rename to java/forstjni/comparatorjnicallback.cc index d354b40b8..775399223 100644 --- a/java/rocksjni/comparatorjnicallback.cc +++ b/java/forstjni/comparatorjnicallback.cc @@ -6,9 +6,9 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::Comparator. -#include "rocksjni/comparatorjnicallback.h" +#include "forstjni/comparatorjnicallback.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { ComparatorJniCallback::ComparatorJniCallback( diff --git a/java/rocksjni/comparatorjnicallback.h b/java/forstjni/comparatorjnicallback.h similarity index 99% rename from java/rocksjni/comparatorjnicallback.h rename to java/forstjni/comparatorjnicallback.h index 034c0d5d7..671d2a3a8 100644 --- a/java/rocksjni/comparatorjnicallback.h +++ b/java/forstjni/comparatorjnicallback.h @@ -17,7 +17,7 @@ #include "port/port.h" #include "rocksdb/comparator.h" #include "rocksdb/slice.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" #include "util/thread_local.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/compression_options.cc b/java/forstjni/compression_options.cc similarity index 73% rename from java/rocksjni/compression_options.cc rename to java/forstjni/compression_options.cc index 53f240560..702dcb8de 100644 --- a/java/rocksjni/compression_options.cc +++ b/java/forstjni/compression_options.cc @@ -8,27 +8,27 @@ #include -#include "include/org_rocksdb_CompressionOptions.h" +#include "include/org_forstdb_CompressionOptions.h" #include "rocksdb/advanced_options.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: newCompressionOptions * Signature: ()J */ -jlong Java_org_rocksdb_CompressionOptions_newCompressionOptions(JNIEnv*, +jlong Java_org_forstdb_CompressionOptions_newCompressionOptions(JNIEnv*, jclass) { const auto* opt = new ROCKSDB_NAMESPACE::CompressionOptions(); return GET_CPLUSPLUS_POINTER(opt); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: setWindowBits * Signature: (JI)V */ -void Java_org_rocksdb_CompressionOptions_setWindowBits(JNIEnv*, jobject, +void Java_org_forstdb_CompressionOptions_setWindowBits(JNIEnv*, jobject, jlong jhandle, jint jwindow_bits) { auto* opt = reinterpret_cast(jhandle); @@ -36,44 +36,44 @@ void Java_org_rocksdb_CompressionOptions_setWindowBits(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: windowBits * Signature: (J)I */ -jint Java_org_rocksdb_CompressionOptions_windowBits(JNIEnv*, jobject, +jint Java_org_forstdb_CompressionOptions_windowBits(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->window_bits); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: setLevel * Signature: (JI)V */ -void Java_org_rocksdb_CompressionOptions_setLevel(JNIEnv*, jobject, +void Java_org_forstdb_CompressionOptions_setLevel(JNIEnv*, jobject, jlong jhandle, jint jlevel) { auto* opt = reinterpret_cast(jhandle); opt->level = static_cast(jlevel); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: level * Signature: (J)I */ -jint Java_org_rocksdb_CompressionOptions_level(JNIEnv*, jobject, +jint Java_org_forstdb_CompressionOptions_level(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->level); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: setStrategy * Signature: (JI)V */ -void Java_org_rocksdb_CompressionOptions_setStrategy(JNIEnv*, jobject, +void Java_org_forstdb_CompressionOptions_setStrategy(JNIEnv*, jobject, jlong jhandle, jint jstrategy) { auto* opt = reinterpret_cast(jhandle); @@ -81,22 +81,22 @@ void Java_org_rocksdb_CompressionOptions_setStrategy(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: strategy * Signature: (J)I */ -jint Java_org_rocksdb_CompressionOptions_strategy(JNIEnv*, jobject, +jint Java_org_forstdb_CompressionOptions_strategy(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->strategy); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: setMaxDictBytes * Signature: (JI)V */ -void Java_org_rocksdb_CompressionOptions_setMaxDictBytes(JNIEnv*, jobject, +void Java_org_forstdb_CompressionOptions_setMaxDictBytes(JNIEnv*, jobject, jlong jhandle, jint jmax_dict_bytes) { auto* opt = reinterpret_cast(jhandle); @@ -104,77 +104,77 @@ void Java_org_rocksdb_CompressionOptions_setMaxDictBytes(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: maxDictBytes * Signature: (J)I */ -jint Java_org_rocksdb_CompressionOptions_maxDictBytes(JNIEnv*, jobject, +jint Java_org_forstdb_CompressionOptions_maxDictBytes(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_dict_bytes); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: setZstdMaxTrainBytes * Signature: (JI)V */ -void Java_org_rocksdb_CompressionOptions_setZstdMaxTrainBytes( +void Java_org_forstdb_CompressionOptions_setZstdMaxTrainBytes( JNIEnv*, jobject, jlong jhandle, jint jzstd_max_train_bytes) { auto* opt = reinterpret_cast(jhandle); opt->zstd_max_train_bytes = static_cast(jzstd_max_train_bytes); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: zstdMaxTrainBytes * Signature: (J)I */ -jint Java_org_rocksdb_CompressionOptions_zstdMaxTrainBytes(JNIEnv*, jobject, +jint Java_org_forstdb_CompressionOptions_zstdMaxTrainBytes(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->zstd_max_train_bytes); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: setMaxDictBufferBytes * Signature: (JJ)V */ -void Java_org_rocksdb_CompressionOptions_setMaxDictBufferBytes( +void Java_org_forstdb_CompressionOptions_setMaxDictBufferBytes( JNIEnv*, jobject, jlong jhandle, jlong jmax_dict_buffer_bytes) { auto* opt = reinterpret_cast(jhandle); opt->max_dict_buffer_bytes = static_cast(jmax_dict_buffer_bytes); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: maxDictBufferBytes * Signature: (J)J */ -jlong Java_org_rocksdb_CompressionOptions_maxDictBufferBytes(JNIEnv*, jobject, +jlong Java_org_forstdb_CompressionOptions_maxDictBufferBytes(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_dict_buffer_bytes); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: setZstdMaxTrainBytes * Signature: (JZ)V */ -void Java_org_rocksdb_CompressionOptions_setUseZstdDictTrainer( +void Java_org_forstdb_CompressionOptions_setUseZstdDictTrainer( JNIEnv*, jobject, jlong jhandle, jboolean juse_zstd_dict_trainer) { auto* opt = reinterpret_cast(jhandle); opt->use_zstd_dict_trainer = juse_zstd_dict_trainer == JNI_TRUE; } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: zstdMaxTrainBytes * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompressionOptions_useZstdDictTrainer(JNIEnv*, +jboolean Java_org_forstdb_CompressionOptions_useZstdDictTrainer(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -182,11 +182,11 @@ jboolean Java_org_rocksdb_CompressionOptions_useZstdDictTrainer(JNIEnv*, } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: setEnabled * Signature: (JZ)V */ -void Java_org_rocksdb_CompressionOptions_setEnabled(JNIEnv*, jobject, +void Java_org_forstdb_CompressionOptions_setEnabled(JNIEnv*, jobject, jlong jhandle, jboolean jenabled) { auto* opt = reinterpret_cast(jhandle); @@ -194,21 +194,21 @@ void Java_org_rocksdb_CompressionOptions_setEnabled(JNIEnv*, jobject, } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: enabled * Signature: (J)Z */ -jboolean Java_org_rocksdb_CompressionOptions_enabled(JNIEnv*, jobject, +jboolean Java_org_forstdb_CompressionOptions_enabled(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->enabled); } /* - * Class: org_rocksdb_CompressionOptions + * Class: org_forstdb_CompressionOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_CompressionOptions_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_CompressionOptions_disposeInternal(JNIEnv*, jobject, jlong jhandle) { delete reinterpret_cast(jhandle); } diff --git a/java/rocksjni/concurrent_task_limiter.cc b/java/forstjni/concurrent_task_limiter.cc similarity index 75% rename from java/rocksjni/concurrent_task_limiter.cc rename to java/forstjni/concurrent_task_limiter.cc index 0b0b2d271..0c9b08c27 100644 --- a/java/rocksjni/concurrent_task_limiter.cc +++ b/java/forstjni/concurrent_task_limiter.cc @@ -11,16 +11,16 @@ #include #include -#include "include/org_rocksdb_ConcurrentTaskLimiterImpl.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_ConcurrentTaskLimiterImpl.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Class: org_forstdb_ConcurrentTaskLimiterImpl * Method: newConcurrentTaskLimiterImpl0 * Signature: (Ljava/lang/String;I)J */ -jlong Java_org_rocksdb_ConcurrentTaskLimiterImpl_newConcurrentTaskLimiterImpl0( +jlong Java_org_forstdb_ConcurrentTaskLimiterImpl_newConcurrentTaskLimiterImpl0( JNIEnv* env, jclass, jstring jname, jint limit) { jboolean has_exception = JNI_FALSE; std::string name = @@ -36,11 +36,11 @@ jlong Java_org_rocksdb_ConcurrentTaskLimiterImpl_newConcurrentTaskLimiterImpl0( } /* - * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Class: org_forstdb_ConcurrentTaskLimiterImpl * Method: name * Signature: (J)Ljava/lang/String; */ -jstring Java_org_rocksdb_ConcurrentTaskLimiterImpl_name(JNIEnv* env, jclass, +jstring Java_org_forstdb_ConcurrentTaskLimiterImpl_name(JNIEnv* env, jclass, jlong handle) { const auto& limiter = *reinterpret_cast< std::shared_ptr*>(handle); @@ -48,11 +48,11 @@ jstring Java_org_rocksdb_ConcurrentTaskLimiterImpl_name(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Class: org_forstdb_ConcurrentTaskLimiterImpl * Method: setMaxOutstandingTask * Signature: (JI)V */ -void Java_org_rocksdb_ConcurrentTaskLimiterImpl_setMaxOutstandingTask( +void Java_org_forstdb_ConcurrentTaskLimiterImpl_setMaxOutstandingTask( JNIEnv*, jclass, jlong handle, jint max_outstanding_task) { const auto& limiter = *reinterpret_cast< std::shared_ptr*>(handle); @@ -60,11 +60,11 @@ void Java_org_rocksdb_ConcurrentTaskLimiterImpl_setMaxOutstandingTask( } /* - * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Class: org_forstdb_ConcurrentTaskLimiterImpl * Method: resetMaxOutstandingTask * Signature: (J)V */ -void Java_org_rocksdb_ConcurrentTaskLimiterImpl_resetMaxOutstandingTask( +void Java_org_forstdb_ConcurrentTaskLimiterImpl_resetMaxOutstandingTask( JNIEnv*, jclass, jlong handle) { const auto& limiter = *reinterpret_cast< std::shared_ptr*>(handle); @@ -72,11 +72,11 @@ void Java_org_rocksdb_ConcurrentTaskLimiterImpl_resetMaxOutstandingTask( } /* - * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Class: org_forstdb_ConcurrentTaskLimiterImpl * Method: outstandingTask * Signature: (J)I */ -jint Java_org_rocksdb_ConcurrentTaskLimiterImpl_outstandingTask(JNIEnv*, jclass, +jint Java_org_forstdb_ConcurrentTaskLimiterImpl_outstandingTask(JNIEnv*, jclass, jlong handle) { const auto& limiter = *reinterpret_cast< std::shared_ptr*>(handle); @@ -84,11 +84,11 @@ jint Java_org_rocksdb_ConcurrentTaskLimiterImpl_outstandingTask(JNIEnv*, jclass, } /* - * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Class: org_forstdb_ConcurrentTaskLimiterImpl * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_ConcurrentTaskLimiterImpl_disposeInternal(JNIEnv*, +void Java_org_forstdb_ConcurrentTaskLimiterImpl_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* ptr = reinterpret_cast< diff --git a/java/rocksjni/config_options.cc b/java/forstjni/config_options.cc similarity index 76% rename from java/rocksjni/config_options.cc rename to java/forstjni/config_options.cc index 55a9cbb66..dd11ab813 100644 --- a/java/rocksjni/config_options.cc +++ b/java/forstjni/config_options.cc @@ -9,17 +9,17 @@ #include -#include "include/org_rocksdb_ConfigOptions.h" +#include "include/org_forstdb_ConfigOptions.h" #include "rocksdb/convenience.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_ConfigOptions + * Class: org_forstdb_ConfigOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_ConfigOptions_disposeInternal(JNIEnv *, jobject, +void Java_org_forstdb_ConfigOptions_disposeInternal(JNIEnv *, jobject, jlong jhandle) { auto *co = reinterpret_cast(jhandle); assert(co != nullptr); @@ -27,21 +27,21 @@ void Java_org_rocksdb_ConfigOptions_disposeInternal(JNIEnv *, jobject, } /* - * Class: org_rocksdb_ConfigOptions + * Class: org_forstdb_ConfigOptions * Method: newConfigOptions * Signature: ()J */ -jlong Java_org_rocksdb_ConfigOptions_newConfigOptions(JNIEnv *, jclass) { +jlong Java_org_forstdb_ConfigOptions_newConfigOptions(JNIEnv *, jclass) { auto *cfg_opt = new ROCKSDB_NAMESPACE::ConfigOptions(); return GET_CPLUSPLUS_POINTER(cfg_opt); } /* - * Class: org_rocksdb_ConfigOptions + * Class: org_forstdb_ConfigOptions * Method: setEnv * Signature: (JJ;)V */ -void Java_org_rocksdb_ConfigOptions_setEnv(JNIEnv *, jclass, jlong handle, +void Java_org_forstdb_ConfigOptions_setEnv(JNIEnv *, jclass, jlong handle, jlong rocksdb_env_handle) { auto *cfg_opt = reinterpret_cast(handle); auto *rocksdb_env = @@ -50,11 +50,11 @@ void Java_org_rocksdb_ConfigOptions_setEnv(JNIEnv *, jclass, jlong handle, } /* - * Class: org_rocksdb_ConfigOptions + * Class: org_forstdb_ConfigOptions * Method: setDelimiter * Signature: (JLjava/lang/String;)V */ -void Java_org_rocksdb_ConfigOptions_setDelimiter(JNIEnv *env, jclass, +void Java_org_forstdb_ConfigOptions_setDelimiter(JNIEnv *env, jclass, jlong handle, jstring s) { auto *cfg_opt = reinterpret_cast(handle); const char *delim = env->GetStringUTFChars(s, nullptr); @@ -67,11 +67,11 @@ void Java_org_rocksdb_ConfigOptions_setDelimiter(JNIEnv *env, jclass, } /* - * Class: org_rocksdb_ConfigOptions + * Class: org_forstdb_ConfigOptions * Method: setIgnoreUnknownOptions * Signature: (JZ)V */ -void Java_org_rocksdb_ConfigOptions_setIgnoreUnknownOptions(JNIEnv *, jclass, +void Java_org_forstdb_ConfigOptions_setIgnoreUnknownOptions(JNIEnv *, jclass, jlong handle, jboolean b) { auto *cfg_opt = reinterpret_cast(handle); @@ -79,11 +79,11 @@ void Java_org_rocksdb_ConfigOptions_setIgnoreUnknownOptions(JNIEnv *, jclass, } /* - * Class: org_rocksdb_ConfigOptions + * Class: org_forstdb_ConfigOptions * Method: setInputStringsEscaped * Signature: (JZ)V */ -void Java_org_rocksdb_ConfigOptions_setInputStringsEscaped(JNIEnv *, jclass, +void Java_org_forstdb_ConfigOptions_setInputStringsEscaped(JNIEnv *, jclass, jlong handle, jboolean b) { auto *cfg_opt = reinterpret_cast(handle); @@ -91,11 +91,11 @@ void Java_org_rocksdb_ConfigOptions_setInputStringsEscaped(JNIEnv *, jclass, } /* - * Class: org_rocksdb_ConfigOptions + * Class: org_forstdb_ConfigOptions * Method: setSanityLevel * Signature: (JI)V */ -void Java_org_rocksdb_ConfigOptions_setSanityLevel(JNIEnv *, jclass, +void Java_org_forstdb_ConfigOptions_setSanityLevel(JNIEnv *, jclass, jlong handle, jbyte level) { auto *cfg_opt = reinterpret_cast(handle); cfg_opt->sanity_level = diff --git a/java/rocksjni/cplusplus_to_java_convert.h b/java/forstjni/cplusplus_to_java_convert.h similarity index 100% rename from java/rocksjni/cplusplus_to_java_convert.h rename to java/forstjni/cplusplus_to_java_convert.h diff --git a/java/rocksjni/env.cc b/java/forstjni/env.cc similarity index 78% rename from java/rocksjni/env.cc rename to java/forstjni/env.cc index bb739fe2b..32a9bbe01 100644 --- a/java/rocksjni/env.cc +++ b/java/forstjni/env.cc @@ -12,28 +12,28 @@ #include -#include "include/org_rocksdb_Env.h" -#include "include/org_rocksdb_RocksEnv.h" -#include "include/org_rocksdb_RocksMemEnv.h" -#include "include/org_rocksdb_TimedEnv.h" +#include "include/org_forstdb_Env.h" +#include "include/org_forstdb_RocksEnv.h" +#include "include/org_forstdb_RocksMemEnv.h" +#include "include/org_forstdb_TimedEnv.h" #include "portal.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_Env + * Class: org_forstdb_Env * Method: getDefaultEnvInternal * Signature: ()J */ -jlong Java_org_rocksdb_Env_getDefaultEnvInternal(JNIEnv*, jclass) { +jlong Java_org_forstdb_Env_getDefaultEnvInternal(JNIEnv*, jclass) { return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::Env::Default()); } /* - * Class: org_rocksdb_RocksEnv + * Class: org_forstdb_RocksEnv * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_RocksEnv_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_RocksEnv_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* e = reinterpret_cast(jhandle); assert(e != nullptr); @@ -41,11 +41,11 @@ void Java_org_rocksdb_RocksEnv_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Env + * Class: org_forstdb_Env * Method: setBackgroundThreads * Signature: (JIB)V */ -void Java_org_rocksdb_Env_setBackgroundThreads(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Env_setBackgroundThreads(JNIEnv*, jobject, jlong jhandle, jint jnum, jbyte jpriority_value) { auto* rocks_env = reinterpret_cast(jhandle); @@ -55,11 +55,11 @@ void Java_org_rocksdb_Env_setBackgroundThreads(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Env + * Class: org_forstdb_Env * Method: getBackgroundThreads * Signature: (JB)I */ -jint Java_org_rocksdb_Env_getBackgroundThreads(JNIEnv*, jobject, jlong jhandle, +jint Java_org_forstdb_Env_getBackgroundThreads(JNIEnv*, jobject, jlong jhandle, jbyte jpriority_value) { auto* rocks_env = reinterpret_cast(jhandle); const int num = rocks_env->GetBackgroundThreads( @@ -68,11 +68,11 @@ jint Java_org_rocksdb_Env_getBackgroundThreads(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Env + * Class: org_forstdb_Env * Method: getThreadPoolQueueLen * Signature: (JB)I */ -jint Java_org_rocksdb_Env_getThreadPoolQueueLen(JNIEnv*, jobject, jlong jhandle, +jint Java_org_forstdb_Env_getThreadPoolQueueLen(JNIEnv*, jobject, jlong jhandle, jbyte jpriority_value) { auto* rocks_env = reinterpret_cast(jhandle); const int queue_len = rocks_env->GetThreadPoolQueueLen( @@ -81,11 +81,11 @@ jint Java_org_rocksdb_Env_getThreadPoolQueueLen(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Env + * Class: org_forstdb_Env * Method: incBackgroundThreadsIfNeeded * Signature: (JIB)V */ -void Java_org_rocksdb_Env_incBackgroundThreadsIfNeeded(JNIEnv*, jobject, +void Java_org_forstdb_Env_incBackgroundThreadsIfNeeded(JNIEnv*, jobject, jlong jhandle, jint jnum, jbyte jpriority_value) { auto* rocks_env = reinterpret_cast(jhandle); @@ -95,11 +95,11 @@ void Java_org_rocksdb_Env_incBackgroundThreadsIfNeeded(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Env + * Class: org_forstdb_Env * Method: lowerThreadPoolIOPriority * Signature: (JB)V */ -void Java_org_rocksdb_Env_lowerThreadPoolIOPriority(JNIEnv*, jobject, +void Java_org_forstdb_Env_lowerThreadPoolIOPriority(JNIEnv*, jobject, jlong jhandle, jbyte jpriority_value) { auto* rocks_env = reinterpret_cast(jhandle); @@ -108,11 +108,11 @@ void Java_org_rocksdb_Env_lowerThreadPoolIOPriority(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Env + * Class: org_forstdb_Env * Method: lowerThreadPoolCPUPriority * Signature: (JB)V */ -void Java_org_rocksdb_Env_lowerThreadPoolCPUPriority(JNIEnv*, jobject, +void Java_org_forstdb_Env_lowerThreadPoolCPUPriority(JNIEnv*, jobject, jlong jhandle, jbyte jpriority_value) { auto* rocks_env = reinterpret_cast(jhandle); @@ -121,11 +121,11 @@ void Java_org_rocksdb_Env_lowerThreadPoolCPUPriority(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Env + * Class: org_forstdb_Env * Method: getThreadList - * Signature: (J)[Lorg/rocksdb/ThreadStatus; + * Signature: (J)[Lorg/forstdb/ThreadStatus; */ -jobjectArray Java_org_rocksdb_Env_getThreadList(JNIEnv* env, jobject, +jobjectArray Java_org_forstdb_Env_getThreadList(JNIEnv* env, jobject, jlong jhandle) { auto* rocks_env = reinterpret_cast(jhandle); std::vector thread_status; @@ -159,22 +159,22 @@ jobjectArray Java_org_rocksdb_Env_getThreadList(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksMemEnv + * Class: org_forstdb_RocksMemEnv * Method: createMemEnv * Signature: (J)J */ -jlong Java_org_rocksdb_RocksMemEnv_createMemEnv(JNIEnv*, jclass, +jlong Java_org_forstdb_RocksMemEnv_createMemEnv(JNIEnv*, jclass, jlong jbase_env_handle) { auto* base_env = reinterpret_cast(jbase_env_handle); return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::NewMemEnv(base_env)); } /* - * Class: org_rocksdb_RocksMemEnv + * Class: org_forstdb_RocksMemEnv * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_RocksMemEnv_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_RocksMemEnv_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* e = reinterpret_cast(jhandle); assert(e != nullptr); @@ -182,22 +182,22 @@ void Java_org_rocksdb_RocksMemEnv_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_TimedEnv + * Class: org_forstdb_TimedEnv * Method: createTimedEnv * Signature: (J)J */ -jlong Java_org_rocksdb_TimedEnv_createTimedEnv(JNIEnv*, jclass, +jlong Java_org_forstdb_TimedEnv_createTimedEnv(JNIEnv*, jclass, jlong jbase_env_handle) { auto* base_env = reinterpret_cast(jbase_env_handle); return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::NewTimedEnv(base_env)); } /* - * Class: org_rocksdb_TimedEnv + * Class: org_forstdb_TimedEnv * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_TimedEnv_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_TimedEnv_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* e = reinterpret_cast(jhandle); assert(e != nullptr); diff --git a/java/rocksjni/env_flink.cc b/java/forstjni/env_flink.cc similarity index 87% rename from java/rocksjni/env_flink.cc rename to java/forstjni/env_flink.cc index 55d2edca5..c3fee7690 100644 --- a/java/rocksjni/env_flink.cc +++ b/java/forstjni/env_flink.cc @@ -20,16 +20,17 @@ #include -#include "include/org_rocksdb_FlinkEnv.h" -#include "java/rocksjni/portal.h" +#include "include/org_forstdb_FlinkEnv.h" + +#include "java/forstjni/portal.h" #include "rocksdb/env.h" /* - * Class: org_rocksdb_FlinkEnv + * Class: org_forstdb_FlinkEnv * Method: createFlinkEnv * Signature: (Ljava/lang/String;)J */ -jlong Java_org_rocksdb_FlinkEnv_createFlinkEnv(JNIEnv* env, jclass, +jlong Java_org_forstdb_FlinkEnv_createFlinkEnv(JNIEnv* env, jclass, jstring base_path) { jboolean has_exception = JNI_FALSE; auto path = @@ -50,11 +51,11 @@ jlong Java_org_rocksdb_FlinkEnv_createFlinkEnv(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_FlinkEnv + * Class: org_forstdb_FlinkEnv * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_FlinkEnv_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_FlinkEnv_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* handle = reinterpret_cast(jhandle); assert(handle != nullptr); diff --git a/java/rocksjni/env_flink_test_suite.cc b/java/forstjni/env_flink_test_suite.cc similarity index 84% rename from java/rocksjni/env_flink_test_suite.cc rename to java/forstjni/env_flink_test_suite.cc index 5e66ca746..529f95018 100644 --- a/java/rocksjni/env_flink_test_suite.cc +++ b/java/forstjni/env_flink_test_suite.cc @@ -20,15 +20,15 @@ #include -#include "include/org_rocksdb_EnvFlinkTestSuite.h" -#include "java/rocksjni/portal.h" +#include "include/org_forstdb_EnvFlinkTestSuite.h" +#include "java/forstjni/portal.h" /* - * Class: org_rocksdb_EnvFlinkTestSuite + * Class: org_forstdb_EnvFlinkTestSuite * Method: buildNativeObject * Signature: (Ljava/lang/String;)J */ -jlong Java_org_rocksdb_EnvFlinkTestSuite_buildNativeObject(JNIEnv* env, jobject, +jlong Java_org_forstdb_EnvFlinkTestSuite_buildNativeObject(JNIEnv* env, jobject, jstring basePath) { jboolean has_exception = JNI_FALSE; auto path = @@ -43,11 +43,11 @@ jlong Java_org_rocksdb_EnvFlinkTestSuite_buildNativeObject(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_EnvFlinkTestSuite + * Class: org_forstdb_EnvFlinkTestSuite * Method: runAllTestSuites * Signature: (J)V */ -JNIEXPORT void JNICALL Java_org_rocksdb_EnvFlinkTestSuite_runAllTestSuites( +JNIEXPORT void JNICALL Java_org_forstdb_EnvFlinkTestSuite_runAllTestSuites( JNIEnv* jniEnv, jobject, jlong objectHandle) { auto env_flink_test_suites = reinterpret_cast(objectHandle); @@ -61,11 +61,11 @@ JNIEXPORT void JNICALL Java_org_rocksdb_EnvFlinkTestSuite_runAllTestSuites( } /* - * Class: org_rocksdb_EnvFlinkTestSuite + * Class: org_forstdb_EnvFlinkTestSuite * Method: disposeInternal * Signature: (J)V */ -JNIEXPORT void JNICALL Java_org_rocksdb_EnvFlinkTestSuite_disposeInternal( +JNIEXPORT void JNICALL Java_org_forstdb_EnvFlinkTestSuite_disposeInternal( JNIEnv*, jobject, jlong objectHandle) { auto test_suites = reinterpret_cast(objectHandle); diff --git a/java/rocksjni/env_options.cc b/java/forstjni/env_options.cc similarity index 72% rename from java/rocksjni/env_options.cc rename to java/forstjni/env_options.cc index 3237e2775..a0d6b1158 100644 --- a/java/rocksjni/env_options.cc +++ b/java/forstjni/env_options.cc @@ -9,9 +9,9 @@ #include -#include "include/org_rocksdb_EnvOptions.h" +#include "include/org_forstdb_EnvOptions.h" #include "rocksdb/env.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" #define ENV_OPTIONS_SET_BOOL(_jhandle, _opt) \ reinterpret_cast(_jhandle)->_opt = \ @@ -29,21 +29,21 @@ reinterpret_cast(_jhandle)->_opt /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: newEnvOptions * Signature: ()J */ -jlong Java_org_rocksdb_EnvOptions_newEnvOptions__(JNIEnv *, jclass) { +jlong Java_org_forstdb_EnvOptions_newEnvOptions__(JNIEnv *, jclass) { auto *env_opt = new ROCKSDB_NAMESPACE::EnvOptions(); return GET_CPLUSPLUS_POINTER(env_opt); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: newEnvOptions * Signature: (J)J */ -jlong Java_org_rocksdb_EnvOptions_newEnvOptions__J(JNIEnv *, jclass, +jlong Java_org_forstdb_EnvOptions_newEnvOptions__J(JNIEnv *, jclass, jlong jdboptions_handle) { auto *db_options = reinterpret_cast(jdboptions_handle); @@ -52,11 +52,11 @@ jlong Java_org_rocksdb_EnvOptions_newEnvOptions__J(JNIEnv *, jclass, } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_EnvOptions_disposeInternal(JNIEnv *, jobject, +void Java_org_forstdb_EnvOptions_disposeInternal(JNIEnv *, jobject, jlong jhandle) { auto *eo = reinterpret_cast(jhandle); assert(eo != nullptr); @@ -64,237 +64,237 @@ void Java_org_rocksdb_EnvOptions_disposeInternal(JNIEnv *, jobject, } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setUseMmapReads * Signature: (JZ)V */ -void Java_org_rocksdb_EnvOptions_setUseMmapReads(JNIEnv *, jobject, +void Java_org_forstdb_EnvOptions_setUseMmapReads(JNIEnv *, jobject, jlong jhandle, jboolean use_mmap_reads) { ENV_OPTIONS_SET_BOOL(jhandle, use_mmap_reads); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: useMmapReads * Signature: (J)Z */ -jboolean Java_org_rocksdb_EnvOptions_useMmapReads(JNIEnv *, jobject, +jboolean Java_org_forstdb_EnvOptions_useMmapReads(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, use_mmap_reads); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setUseMmapWrites * Signature: (JZ)V */ -void Java_org_rocksdb_EnvOptions_setUseMmapWrites(JNIEnv *, jobject, +void Java_org_forstdb_EnvOptions_setUseMmapWrites(JNIEnv *, jobject, jlong jhandle, jboolean use_mmap_writes) { ENV_OPTIONS_SET_BOOL(jhandle, use_mmap_writes); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: useMmapWrites * Signature: (J)Z */ -jboolean Java_org_rocksdb_EnvOptions_useMmapWrites(JNIEnv *, jobject, +jboolean Java_org_forstdb_EnvOptions_useMmapWrites(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, use_mmap_writes); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setUseDirectReads * Signature: (JZ)V */ -void Java_org_rocksdb_EnvOptions_setUseDirectReads(JNIEnv *, jobject, +void Java_org_forstdb_EnvOptions_setUseDirectReads(JNIEnv *, jobject, jlong jhandle, jboolean use_direct_reads) { ENV_OPTIONS_SET_BOOL(jhandle, use_direct_reads); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: useDirectReads * Signature: (J)Z */ -jboolean Java_org_rocksdb_EnvOptions_useDirectReads(JNIEnv *, jobject, +jboolean Java_org_forstdb_EnvOptions_useDirectReads(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, use_direct_reads); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setUseDirectWrites * Signature: (JZ)V */ -void Java_org_rocksdb_EnvOptions_setUseDirectWrites( +void Java_org_forstdb_EnvOptions_setUseDirectWrites( JNIEnv *, jobject, jlong jhandle, jboolean use_direct_writes) { ENV_OPTIONS_SET_BOOL(jhandle, use_direct_writes); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: useDirectWrites * Signature: (J)Z */ -jboolean Java_org_rocksdb_EnvOptions_useDirectWrites(JNIEnv *, jobject, +jboolean Java_org_forstdb_EnvOptions_useDirectWrites(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, use_direct_writes); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setAllowFallocate * Signature: (JZ)V */ -void Java_org_rocksdb_EnvOptions_setAllowFallocate(JNIEnv *, jobject, +void Java_org_forstdb_EnvOptions_setAllowFallocate(JNIEnv *, jobject, jlong jhandle, jboolean allow_fallocate) { ENV_OPTIONS_SET_BOOL(jhandle, allow_fallocate); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: allowFallocate * Signature: (J)Z */ -jboolean Java_org_rocksdb_EnvOptions_allowFallocate(JNIEnv *, jobject, +jboolean Java_org_forstdb_EnvOptions_allowFallocate(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, allow_fallocate); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setSetFdCloexec * Signature: (JZ)V */ -void Java_org_rocksdb_EnvOptions_setSetFdCloexec(JNIEnv *, jobject, +void Java_org_forstdb_EnvOptions_setSetFdCloexec(JNIEnv *, jobject, jlong jhandle, jboolean set_fd_cloexec) { ENV_OPTIONS_SET_BOOL(jhandle, set_fd_cloexec); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setFdCloexec * Signature: (J)Z */ -jboolean Java_org_rocksdb_EnvOptions_setFdCloexec(JNIEnv *, jobject, +jboolean Java_org_forstdb_EnvOptions_setFdCloexec(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, set_fd_cloexec); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setBytesPerSync * Signature: (JJ)V */ -void Java_org_rocksdb_EnvOptions_setBytesPerSync(JNIEnv *, jobject, +void Java_org_forstdb_EnvOptions_setBytesPerSync(JNIEnv *, jobject, jlong jhandle, jlong bytes_per_sync) { ENV_OPTIONS_SET_UINT64_T(jhandle, bytes_per_sync); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: bytesPerSync * Signature: (J)J */ -jlong Java_org_rocksdb_EnvOptions_bytesPerSync(JNIEnv *, jobject, +jlong Java_org_forstdb_EnvOptions_bytesPerSync(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, bytes_per_sync); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setFallocateWithKeepSize * Signature: (JZ)V */ -void Java_org_rocksdb_EnvOptions_setFallocateWithKeepSize( +void Java_org_forstdb_EnvOptions_setFallocateWithKeepSize( JNIEnv *, jobject, jlong jhandle, jboolean fallocate_with_keep_size) { ENV_OPTIONS_SET_BOOL(jhandle, fallocate_with_keep_size); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: fallocateWithKeepSize * Signature: (J)Z */ -jboolean Java_org_rocksdb_EnvOptions_fallocateWithKeepSize(JNIEnv *, jobject, +jboolean Java_org_forstdb_EnvOptions_fallocateWithKeepSize(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, fallocate_with_keep_size); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setCompactionReadaheadSize * Signature: (JJ)V */ -void Java_org_rocksdb_EnvOptions_setCompactionReadaheadSize( +void Java_org_forstdb_EnvOptions_setCompactionReadaheadSize( JNIEnv *, jobject, jlong jhandle, jlong compaction_readahead_size) { ENV_OPTIONS_SET_SIZE_T(jhandle, compaction_readahead_size); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: compactionReadaheadSize * Signature: (J)J */ -jlong Java_org_rocksdb_EnvOptions_compactionReadaheadSize(JNIEnv *, jobject, +jlong Java_org_forstdb_EnvOptions_compactionReadaheadSize(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, compaction_readahead_size); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setRandomAccessMaxBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_EnvOptions_setRandomAccessMaxBufferSize( +void Java_org_forstdb_EnvOptions_setRandomAccessMaxBufferSize( JNIEnv *, jobject, jlong jhandle, jlong random_access_max_buffer_size) { ENV_OPTIONS_SET_SIZE_T(jhandle, random_access_max_buffer_size); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: randomAccessMaxBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_EnvOptions_randomAccessMaxBufferSize(JNIEnv *, jobject, +jlong Java_org_forstdb_EnvOptions_randomAccessMaxBufferSize(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, random_access_max_buffer_size); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setWritableFileMaxBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_EnvOptions_setWritableFileMaxBufferSize( +void Java_org_forstdb_EnvOptions_setWritableFileMaxBufferSize( JNIEnv *, jobject, jlong jhandle, jlong writable_file_max_buffer_size) { ENV_OPTIONS_SET_SIZE_T(jhandle, writable_file_max_buffer_size); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: writableFileMaxBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_EnvOptions_writableFileMaxBufferSize(JNIEnv *, jobject, +jlong Java_org_forstdb_EnvOptions_writableFileMaxBufferSize(JNIEnv *, jobject, jlong jhandle) { return ENV_OPTIONS_GET(jhandle, writable_file_max_buffer_size); } /* - * Class: org_rocksdb_EnvOptions + * Class: org_forstdb_EnvOptions * Method: setRateLimiter * Signature: (JJ)V */ -void Java_org_rocksdb_EnvOptions_setRateLimiter(JNIEnv *, jobject, +void Java_org_forstdb_EnvOptions_setRateLimiter(JNIEnv *, jobject, jlong jhandle, jlong rl_handle) { auto *sptr_rate_limiter = diff --git a/java/rocksjni/event_listener.cc b/java/forstjni/event_listener.cc similarity index 74% rename from java/rocksjni/event_listener.cc rename to java/forstjni/event_listener.cc index 965932c9c..2f73b8b01 100644 --- a/java/rocksjni/event_listener.cc +++ b/java/forstjni/event_listener.cc @@ -10,17 +10,17 @@ #include -#include "include/org_rocksdb_AbstractEventListener.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/event_listener_jnicallback.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_AbstractEventListener.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/event_listener_jnicallback.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_AbstractEventListener + * Class: org_forstdb_AbstractEventListener * Method: createNewEventListener * Signature: (J)J */ -jlong Java_org_rocksdb_AbstractEventListener_createNewEventListener( +jlong Java_org_forstdb_AbstractEventListener_createNewEventListener( JNIEnv* env, jobject jobj, jlong jenabled_event_callback_values) { auto enabled_event_callbacks = ROCKSDB_NAMESPACE::EnabledEventCallbackJni::toCppEnabledEventCallbacks( @@ -33,11 +33,11 @@ jlong Java_org_rocksdb_AbstractEventListener_createNewEventListener( } /* - * Class: org_rocksdb_AbstractEventListener + * Class: org_forstdb_AbstractEventListener * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_AbstractEventListener_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_AbstractEventListener_disposeInternal(JNIEnv*, jobject, jlong jhandle) { delete reinterpret_cast*>( jhandle); diff --git a/java/rocksjni/event_listener_jnicallback.cc b/java/forstjni/event_listener_jnicallback.cc similarity index 99% rename from java/rocksjni/event_listener_jnicallback.cc rename to java/forstjni/event_listener_jnicallback.cc index 342d938b4..deb8d65de 100644 --- a/java/rocksjni/event_listener_jnicallback.cc +++ b/java/forstjni/event_listener_jnicallback.cc @@ -6,9 +6,9 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::EventListener. -#include "rocksjni/event_listener_jnicallback.h" +#include "forstjni/event_listener_jnicallback.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { EventListenerJniCallback::EventListenerJniCallback( diff --git a/java/rocksjni/event_listener_jnicallback.h b/java/forstjni/event_listener_jnicallback.h similarity index 99% rename from java/rocksjni/event_listener_jnicallback.h rename to java/forstjni/event_listener_jnicallback.h index f4a235a23..564210d37 100644 --- a/java/rocksjni/event_listener_jnicallback.h +++ b/java/forstjni/event_listener_jnicallback.h @@ -15,7 +15,7 @@ #include #include "rocksdb/listener.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/forstjni/export_import_files_metadatajni.cc b/java/forstjni/export_import_files_metadatajni.cc new file mode 100644 index 000000000..547b49b4c --- /dev/null +++ b/java/forstjni/export_import_files_metadatajni.cc @@ -0,0 +1,22 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "include/org_forstdb_ExportImportFilesMetaData.h" +#include "include/org_forstdb_LiveFileMetaData.h" +#include "forstjni/portal.h" + +/* + * Class: org_forstdb_ExportImportFilesMetaData + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_forstdb_ExportImportFilesMetaData_disposeInternal( + JNIEnv* /*env*/, jobject /*jopt*/, jlong jhandle) { + auto* metadata = + reinterpret_cast(jhandle); + assert(metadata != nullptr); + delete metadata; +} diff --git a/java/rocksjni/filter.cc b/java/forstjni/filter.cc similarity index 76% rename from java/rocksjni/filter.cc rename to java/forstjni/filter.cc index ed22016d2..d07584dfc 100644 --- a/java/rocksjni/filter.cc +++ b/java/forstjni/filter.cc @@ -12,18 +12,18 @@ #include -#include "include/org_rocksdb_BloomFilter.h" -#include "include/org_rocksdb_Filter.h" +#include "include/org_forstdb_BloomFilter.h" +#include "include/org_forstdb_Filter.h" #include "rocksdb/filter_policy.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_BloomFilter + * Class: org_forstdb_BloomFilter * Method: createBloomFilter * Signature: (DZ)J */ -jlong Java_org_rocksdb_BloomFilter_createNewBloomFilter(JNIEnv* /*env*/, +jlong Java_org_forstdb_BloomFilter_createNewBloomFilter(JNIEnv* /*env*/, jclass /*jcls*/, jdouble bits_per_key) { auto* sptr_filter = @@ -33,11 +33,11 @@ jlong Java_org_rocksdb_BloomFilter_createNewBloomFilter(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Filter + * Class: org_forstdb_Filter * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_Filter_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_Filter_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* handle = reinterpret_cast*>( diff --git a/java/rocksjni/flink_compactionfilterjni.cc b/java/forstjni/flink_compactionfilterjni.cc similarity index 71% rename from java/rocksjni/flink_compactionfilterjni.cc rename to java/forstjni/flink_compactionfilterjni.cc index 4354f2878..0f17cd4b3 100644 --- a/java/rocksjni/flink_compactionfilterjni.cc +++ b/java/forstjni/flink_compactionfilterjni.cc @@ -6,14 +6,12 @@ #include #include -#include "include/org_rocksdb_FlinkCompactionFilter.h" +#include "include/org_forstdb_FlinkCompactionFilter.h" #include "loggerjnicallback.h" #include "portal.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" #include "utilities/flink/flink_compaction_filter.h" -namespace flink = ROCKSDB_NAMESPACE::flink; - class JniCallbackBase : public ROCKSDB_NAMESPACE::JniCallback { public: JniCallbackBase(JNIEnv* env, jobject jcallback_obj) @@ -22,8 +20,9 @@ class JniCallbackBase : public ROCKSDB_NAMESPACE::JniCallback { protected: inline void CheckAndRethrowException(JNIEnv* env) const { if (env->ExceptionCheck()) { + jthrowable obj = env->ExceptionOccurred(); env->ExceptionDescribe(); - env->Throw(env->ExceptionOccurred()); + env->Throw(obj); } } }; @@ -40,7 +39,7 @@ class JavaListElementFilter JavaListElementFilter(JNIEnv* env, jobject jlist_filter) : JniCallbackBase(env, jlist_filter) { jclass jclazz = ROCKSDB_NAMESPACE::JavaClass::getJClass( - env, "org/rocksdb/FlinkCompactionFilter$ListElementFilter"); + env, "org/forstdb/FlinkCompactionFilter$ListElementFilter"); if (jclazz == nullptr) { // exception occurred accessing class return; @@ -83,18 +82,19 @@ class JavaListElemenFilterFactory JavaListElemenFilterFactory(JNIEnv* env, jobject jlist_filter_factory) : JniCallbackBase(env, jlist_filter_factory) { jclass jclazz = ROCKSDB_NAMESPACE::JavaClass::getJClass( - env, "org/rocksdb/FlinkCompactionFilter$ListElementFilterFactory"); + env, "org/forstdb/FlinkCompactionFilter$ListElementFilterFactory"); if (jclazz == nullptr) { // exception occurred accessing class return; } m_jcreate_filter_methodid = env->GetMethodID( jclazz, "createListElementFilter", - "()Lorg/rocksdb/FlinkCompactionFilter$ListElementFilter;"); + "()Lorg/forstdb/FlinkCompactionFilter$ListElementFilter;"); assert(m_jcreate_filter_methodid != nullptr); } - flink::FlinkCompactionFilter::ListElementFilter* CreateListElementFilter( + ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter::ListElementFilter* + CreateListElementFilter( std::shared_ptr /*logger*/) const override { jboolean attached_thread = JNI_FALSE; JNIEnv* env = getJniEnv(&attached_thread); @@ -117,7 +117,7 @@ class JavaTimeProvider JavaTimeProvider(JNIEnv* env, jobject jtime_provider) : JniCallbackBase(env, jtime_provider) { jclass jclazz = ROCKSDB_NAMESPACE::JavaClass::getJClass( - env, "org/rocksdb/FlinkCompactionFilter$TimeProvider"); + env, "org/forstdb/FlinkCompactionFilter$TimeProvider"); if (jclazz == nullptr) { // exception occurred accessing class return; @@ -141,16 +141,16 @@ class JavaTimeProvider jmethodID m_jcurrent_timestamp_methodid; }; -static flink::FlinkCompactionFilter::ListElementFilterFactory* -createListElementFilterFactory(JNIEnv* env, jint ji_list_elem_len, - jobject jlist_filter_factory) { - flink::FlinkCompactionFilter::ListElementFilterFactory* list_filter_factory = - nullptr; +static ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter:: + ListElementFilterFactory* + createListElementFilterFactory(JNIEnv* env, jint ji_list_elem_len, + jobject jlist_filter_factory) { + ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter::ListElementFilterFactory* + list_filter_factory = nullptr; if (ji_list_elem_len > 0) { auto fixed_size = static_cast(ji_list_elem_len); - list_filter_factory = - new flink::FlinkCompactionFilter::FixedListElementFilterFactory( - fixed_size, static_cast(0)); + list_filter_factory = new ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter:: + FixedListElementFilterFactory(fixed_size, static_cast(0)); } else if (jlist_filter_factory != nullptr) { list_filter_factory = new JavaListElemenFilterFactory(env, jlist_filter_factory); @@ -159,40 +159,41 @@ createListElementFilterFactory(JNIEnv* env, jint ji_list_elem_len, } /*x - * Class: org_rocksdb_FlinkCompactionFilter + * Class: org_forstdb_FlinkCompactionFilter * Method: createNewFlinkCompactionFilterConfigHolder * Signature: ()J */ -jlong Java_org_rocksdb_FlinkCompactionFilter_createNewFlinkCompactionFilterConfigHolder( +jlong Java_org_forstdb_FlinkCompactionFilter_createNewFlinkCompactionFilterConfigHolder( JNIEnv* /* env */, jclass /* jcls */) { return reinterpret_cast( - new std::shared_ptr( - new flink::FlinkCompactionFilter::ConfigHolder())); + new std::shared_ptr< + ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter::ConfigHolder>( + new ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter::ConfigHolder())); } /* - * Class: org_rocksdb_FlinkCompactionFilter + * Class: org_forstdb_FlinkCompactionFilter * Method: disposeFlinkCompactionFilterConfigHolder * Signature: (J)V */ -void Java_org_rocksdb_FlinkCompactionFilter_disposeFlinkCompactionFilterConfigHolder( +void Java_org_forstdb_FlinkCompactionFilter_disposeFlinkCompactionFilterConfigHolder( JNIEnv* /* env */, jclass /* jcls */, jlong handle) { - auto* config_holder = reinterpret_cast< - std::shared_ptr*>(handle); + auto* config_holder = reinterpret_cast*>(handle); delete config_holder; } /* - * Class: org_rocksdb_FlinkCompactionFilter + * Class: org_forstdb_FlinkCompactionFilter * Method: createNewFlinkCompactionFilter0 * Signature: (JJJ)J */ -jlong Java_org_rocksdb_FlinkCompactionFilter_createNewFlinkCompactionFilter0( +jlong Java_org_forstdb_FlinkCompactionFilter_createNewFlinkCompactionFilter0( JNIEnv* env, jclass /* jcls */, jlong config_holder_handle, jobject jtime_provider, jlong logger_handle) { auto config_holder = - *(reinterpret_cast< - std::shared_ptr*>( + *(reinterpret_cast*>( config_holder_handle)); auto time_provider = new JavaTimeProvider(env, jtime_provider); auto logger = @@ -201,38 +202,41 @@ jlong Java_org_rocksdb_FlinkCompactionFilter_createNewFlinkCompactionFilter0( : *(reinterpret_cast< std::shared_ptr*>( logger_handle)); - return reinterpret_cast(new flink::FlinkCompactionFilter( - config_holder, - std::unique_ptr( - time_provider), - logger)); + return reinterpret_cast( + new ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter( + config_holder, + std::unique_ptr< + ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter::TimeProvider>( + time_provider), + logger)); } /* - * Class: org_rocksdb_FlinkCompactionFilter + * Class: org_forstdb_FlinkCompactionFilter * Method: configureFlinkCompactionFilter - * Signature: (JIIJJILorg/rocksdb/FlinkCompactionFilter$ListElementFilter;)Z + * Signature: (JIIJJILorg/forstdb/FlinkCompactionFilter$ListElementFilter;)Z */ -jboolean Java_org_rocksdb_FlinkCompactionFilter_configureFlinkCompactionFilter( +jboolean Java_org_forstdb_FlinkCompactionFilter_configureFlinkCompactionFilter( JNIEnv* env, jclass /* jcls */, jlong handle, jint ji_state_type, jint ji_timestamp_offset, jlong jl_ttl_milli, jlong jquery_time_after_num_entries, jint ji_list_elem_len, jobject jlist_filter_factory) { auto state_type = - static_cast(ji_state_type); + static_cast( + ji_state_type); auto timestamp_offset = static_cast(ji_timestamp_offset); auto ttl = static_cast(jl_ttl_milli); auto query_time_after_num_entries = static_cast(jquery_time_after_num_entries); auto config_holder = - *(reinterpret_cast< - std::shared_ptr*>( + *(reinterpret_cast*>( handle)); auto list_filter_factory = createListElementFilterFactory( env, ji_list_elem_len, jlist_filter_factory); - auto config = new flink::FlinkCompactionFilter::Config{ + auto config = new ROCKSDB_NAMESPACE::flink::FlinkCompactionFilter::Config{ state_type, timestamp_offset, ttl, query_time_after_num_entries, - std::unique_ptr( - list_filter_factory)}; + std::unique_ptr(list_filter_factory)}; return static_cast(config_holder->Configure(config)); -} \ No newline at end of file +} diff --git a/java/forstjni/hyper_clock_cache.cc b/java/forstjni/hyper_clock_cache.cc new file mode 100644 index 000000000..9fdab09f7 --- /dev/null +++ b/java/forstjni/hyper_clock_cache.cc @@ -0,0 +1,42 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::HyperClockCache. + +#include + +#include "cache/clock_cache.h" +#include "include/org_forstdb_HyperClockCache.h" +#include "forstjni/cplusplus_to_java_convert.h" + +/* + * Class: org_forstdb_HyperClockCache + * Method: newHyperClockCache + * Signature: (JJIZ)J + */ +jlong Java_org_forstdb_HyperClockCache_newHyperClockCache( + JNIEnv*, jclass, jlong capacity, jlong estimatedEntryCharge, + jint numShardBits, jboolean strictCapacityLimit) { + ROCKSDB_NAMESPACE::HyperClockCacheOptions cacheOptions = + ROCKSDB_NAMESPACE::HyperClockCacheOptions( + capacity, estimatedEntryCharge, numShardBits, strictCapacityLimit); + + auto* cache = new std::shared_ptr( + cacheOptions.MakeSharedCache()); + return GET_CPLUSPLUS_POINTER(cache); +} + +/* + * Class: org_forstdb_HyperClockCache + * Method: disposeInternalJni + * Signature: (J)V + */ +void Java_org_forstdb_HyperClockCache_disposeInternalJni(JNIEnv*, jclass, + jlong jhandle) { + auto* hyper_clock_cache = + reinterpret_cast*>(jhandle); + delete hyper_clock_cache; // delete std::shared_ptr +} \ No newline at end of file diff --git a/java/rocksjni/import_column_family_options.cc b/java/forstjni/import_column_family_options.cc similarity index 71% rename from java/rocksjni/import_column_family_options.cc rename to java/forstjni/import_column_family_options.cc index 1a9bded51..3f642871e 100644 --- a/java/rocksjni/import_column_family_options.cc +++ b/java/forstjni/import_column_family_options.cc @@ -6,16 +6,16 @@ #include -#include "include/org_rocksdb_ImportColumnFamilyOptions.h" +#include "include/org_forstdb_ImportColumnFamilyOptions.h" #include "rocksdb/options.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_ImportColumnFamilyOptions + * Class: org_forstdb_ImportColumnFamilyOptions * Method: newImportColumnFamilyOptions * Signature: ()J */ -jlong Java_org_rocksdb_ImportColumnFamilyOptions_newImportColumnFamilyOptions( +jlong Java_org_forstdb_ImportColumnFamilyOptions_newImportColumnFamilyOptions( JNIEnv *, jclass) { ROCKSDB_NAMESPACE::ImportColumnFamilyOptions *opts = new ROCKSDB_NAMESPACE::ImportColumnFamilyOptions(); @@ -23,11 +23,11 @@ jlong Java_org_rocksdb_ImportColumnFamilyOptions_newImportColumnFamilyOptions( } /* - * Class: org_rocksdb_ImportColumnFamilyOptions + * Class: org_forstdb_ImportColumnFamilyOptions * Method: setMoveFiles * Signature: (JZ)V */ -void Java_org_rocksdb_ImportColumnFamilyOptions_setMoveFiles( +void Java_org_forstdb_ImportColumnFamilyOptions_setMoveFiles( JNIEnv *, jobject, jlong jhandle, jboolean jmove_files) { auto *options = reinterpret_cast(jhandle); @@ -35,11 +35,11 @@ void Java_org_rocksdb_ImportColumnFamilyOptions_setMoveFiles( } /* - * Class: org_rocksdb_ImportColumnFamilyOptions + * Class: org_forstdb_ImportColumnFamilyOptions * Method: moveFiles * Signature: (J)Z */ -jboolean Java_org_rocksdb_ImportColumnFamilyOptions_moveFiles(JNIEnv *, jobject, +jboolean Java_org_forstdb_ImportColumnFamilyOptions_moveFiles(JNIEnv *, jobject, jlong jhandle) { auto *options = reinterpret_cast(jhandle); @@ -47,11 +47,11 @@ jboolean Java_org_rocksdb_ImportColumnFamilyOptions_moveFiles(JNIEnv *, jobject, } /* - * Class: org_rocksdb_ImportColumnFamilyOptions + * Class: org_forstdb_ImportColumnFamilyOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_ImportColumnFamilyOptions_disposeInternal(JNIEnv *, +void Java_org_forstdb_ImportColumnFamilyOptions_disposeInternal(JNIEnv *, jobject, jlong jhandle) { delete reinterpret_cast( diff --git a/java/rocksjni/ingest_external_file_options.cc b/java/forstjni/ingest_external_file_options.cc similarity index 73% rename from java/rocksjni/ingest_external_file_options.cc rename to java/forstjni/ingest_external_file_options.cc index 052cf3325..8b87c33ab 100644 --- a/java/rocksjni/ingest_external_file_options.cc +++ b/java/forstjni/ingest_external_file_options.cc @@ -8,27 +8,27 @@ #include -#include "include/org_rocksdb_IngestExternalFileOptions.h" +#include "include/org_forstdb_IngestExternalFileOptions.h" #include "rocksdb/options.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: newIngestExternalFileOptions * Signature: ()J */ -jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__( +jlong Java_org_forstdb_IngestExternalFileOptions_newIngestExternalFileOptions__( JNIEnv*, jclass) { auto* options = new ROCKSDB_NAMESPACE::IngestExternalFileOptions(); return GET_CPLUSPLUS_POINTER(options); } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: newIngestExternalFileOptions * Signature: (ZZZZ)J */ -jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__ZZZZ( +jlong Java_org_forstdb_IngestExternalFileOptions_newIngestExternalFileOptions__ZZZZ( JNIEnv*, jclass, jboolean jmove_files, jboolean jsnapshot_consistency, jboolean jallow_global_seqno, jboolean jallow_blocking_flush) { auto* options = new ROCKSDB_NAMESPACE::IngestExternalFileOptions(); @@ -40,11 +40,11 @@ jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__Z } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: moveFiles * Signature: (J)Z */ -jboolean Java_org_rocksdb_IngestExternalFileOptions_moveFiles(JNIEnv*, jobject, +jboolean Java_org_forstdb_IngestExternalFileOptions_moveFiles(JNIEnv*, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); @@ -52,11 +52,11 @@ jboolean Java_org_rocksdb_IngestExternalFileOptions_moveFiles(JNIEnv*, jobject, } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: setMoveFiles * Signature: (JZ)V */ -void Java_org_rocksdb_IngestExternalFileOptions_setMoveFiles( +void Java_org_forstdb_IngestExternalFileOptions_setMoveFiles( JNIEnv*, jobject, jlong jhandle, jboolean jmove_files) { auto* options = reinterpret_cast(jhandle); @@ -64,11 +64,11 @@ void Java_org_rocksdb_IngestExternalFileOptions_setMoveFiles( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: snapshotConsistency * Signature: (J)Z */ -jboolean Java_org_rocksdb_IngestExternalFileOptions_snapshotConsistency( +jboolean Java_org_forstdb_IngestExternalFileOptions_snapshotConsistency( JNIEnv*, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); @@ -76,11 +76,11 @@ jboolean Java_org_rocksdb_IngestExternalFileOptions_snapshotConsistency( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: setSnapshotConsistency * Signature: (JZ)V */ -void Java_org_rocksdb_IngestExternalFileOptions_setSnapshotConsistency( +void Java_org_forstdb_IngestExternalFileOptions_setSnapshotConsistency( JNIEnv*, jobject, jlong jhandle, jboolean jsnapshot_consistency) { auto* options = reinterpret_cast(jhandle); @@ -88,11 +88,11 @@ void Java_org_rocksdb_IngestExternalFileOptions_setSnapshotConsistency( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: allowGlobalSeqNo * Signature: (J)Z */ -jboolean Java_org_rocksdb_IngestExternalFileOptions_allowGlobalSeqNo( +jboolean Java_org_forstdb_IngestExternalFileOptions_allowGlobalSeqNo( JNIEnv*, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); @@ -100,11 +100,11 @@ jboolean Java_org_rocksdb_IngestExternalFileOptions_allowGlobalSeqNo( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: setAllowGlobalSeqNo * Signature: (JZ)V */ -void Java_org_rocksdb_IngestExternalFileOptions_setAllowGlobalSeqNo( +void Java_org_forstdb_IngestExternalFileOptions_setAllowGlobalSeqNo( JNIEnv*, jobject, jlong jhandle, jboolean jallow_global_seqno) { auto* options = reinterpret_cast(jhandle); @@ -112,11 +112,11 @@ void Java_org_rocksdb_IngestExternalFileOptions_setAllowGlobalSeqNo( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: allowBlockingFlush * Signature: (J)Z */ -jboolean Java_org_rocksdb_IngestExternalFileOptions_allowBlockingFlush( +jboolean Java_org_forstdb_IngestExternalFileOptions_allowBlockingFlush( JNIEnv*, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); @@ -124,11 +124,11 @@ jboolean Java_org_rocksdb_IngestExternalFileOptions_allowBlockingFlush( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: setAllowBlockingFlush * Signature: (JZ)V */ -void Java_org_rocksdb_IngestExternalFileOptions_setAllowBlockingFlush( +void Java_org_forstdb_IngestExternalFileOptions_setAllowBlockingFlush( JNIEnv*, jobject, jlong jhandle, jboolean jallow_blocking_flush) { auto* options = reinterpret_cast(jhandle); @@ -136,11 +136,11 @@ void Java_org_rocksdb_IngestExternalFileOptions_setAllowBlockingFlush( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: ingestBehind * Signature: (J)Z */ -jboolean Java_org_rocksdb_IngestExternalFileOptions_ingestBehind( +jboolean Java_org_forstdb_IngestExternalFileOptions_ingestBehind( JNIEnv*, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); @@ -148,11 +148,11 @@ jboolean Java_org_rocksdb_IngestExternalFileOptions_ingestBehind( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: setIngestBehind * Signature: (JZ)V */ -void Java_org_rocksdb_IngestExternalFileOptions_setIngestBehind( +void Java_org_forstdb_IngestExternalFileOptions_setIngestBehind( JNIEnv*, jobject, jlong jhandle, jboolean jingest_behind) { auto* options = reinterpret_cast(jhandle); @@ -160,12 +160,12 @@ void Java_org_rocksdb_IngestExternalFileOptions_setIngestBehind( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: writeGlobalSeqno * Signature: (J)Z */ JNIEXPORT jboolean JNICALL -Java_org_rocksdb_IngestExternalFileOptions_writeGlobalSeqno(JNIEnv*, jobject, +Java_org_forstdb_IngestExternalFileOptions_writeGlobalSeqno(JNIEnv*, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); @@ -173,12 +173,12 @@ Java_org_rocksdb_IngestExternalFileOptions_writeGlobalSeqno(JNIEnv*, jobject, } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: setWriteGlobalSeqno * Signature: (JZ)V */ JNIEXPORT void JNICALL -Java_org_rocksdb_IngestExternalFileOptions_setWriteGlobalSeqno( +Java_org_forstdb_IngestExternalFileOptions_setWriteGlobalSeqno( JNIEnv*, jobject, jlong jhandle, jboolean jwrite_global_seqno) { auto* options = reinterpret_cast(jhandle); @@ -186,11 +186,11 @@ Java_org_rocksdb_IngestExternalFileOptions_setWriteGlobalSeqno( } /* - * Class: org_rocksdb_IngestExternalFileOptions + * Class: org_forstdb_IngestExternalFileOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_IngestExternalFileOptions_disposeInternal(JNIEnv*, +void Java_org_forstdb_IngestExternalFileOptions_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* options = diff --git a/java/rocksjni/iterator.cc b/java/forstjni/iterator.cc similarity index 82% rename from java/rocksjni/iterator.cc rename to java/forstjni/iterator.cc index 3ddb9778b..c202e5b41 100644 --- a/java/rocksjni/iterator.cc +++ b/java/forstjni/iterator.cc @@ -14,15 +14,15 @@ #include -#include "include/org_rocksdb_RocksIterator.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_RocksIterator.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_RocksIterator_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_RocksIterator_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); @@ -31,64 +31,64 @@ void Java_org_rocksdb_RocksIterator_disposeInternal(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: isValid0 * Signature: (J)Z */ -jboolean Java_org_rocksdb_RocksIterator_isValid0(JNIEnv* /*env*/, +jboolean Java_org_forstdb_RocksIterator_isValid0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { return reinterpret_cast(handle)->Valid(); } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: seekToFirst0 * Signature: (J)V */ -void Java_org_rocksdb_RocksIterator_seekToFirst0(JNIEnv* /*env*/, +void Java_org_forstdb_RocksIterator_seekToFirst0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->SeekToFirst(); } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: seekToLast0 * Signature: (J)V */ -void Java_org_rocksdb_RocksIterator_seekToLast0(JNIEnv* /*env*/, +void Java_org_forstdb_RocksIterator_seekToLast0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->SeekToLast(); } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: next0 * Signature: (J)V */ -void Java_org_rocksdb_RocksIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_RocksIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->Next(); } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: prev0 * Signature: (J)V */ -void Java_org_rocksdb_RocksIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_RocksIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->Prev(); } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: refresh0 * Signature: (J)V */ -void Java_org_rocksdb_RocksIterator_refresh0(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_RocksIterator_refresh0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); ROCKSDB_NAMESPACE::Status s = it->Refresh(); @@ -101,11 +101,11 @@ void Java_org_rocksdb_RocksIterator_refresh0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: seek0 * Signature: (J[BI)V */ -void Java_org_rocksdb_RocksIterator_seek0(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_RocksIterator_seek0(JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -120,11 +120,11 @@ void Java_org_rocksdb_RocksIterator_seek0(JNIEnv* env, jobject /*jobj*/, * the Java wrapper extracts the byte[] and passes it here. * In this case, the buffer offset of the key may be non-zero. * - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: seek0 * Signature: (J[BII)V */ -void Java_org_rocksdb_RocksIterator_seekByteArray0( +void Java_org_forstdb_RocksIterator_seekByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -136,11 +136,11 @@ void Java_org_rocksdb_RocksIterator_seekByteArray0( } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: seekDirect0 * Signature: (JLjava/nio/ByteBuffer;II)V */ -void Java_org_rocksdb_RocksIterator_seekDirect0(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_RocksIterator_seekDirect0(JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { @@ -153,11 +153,11 @@ void Java_org_rocksdb_RocksIterator_seekDirect0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: seekForPrevDirect0 * Signature: (JLjava/nio/ByteBuffer;II)V */ -void Java_org_rocksdb_RocksIterator_seekForPrevDirect0( +void Java_org_forstdb_RocksIterator_seekForPrevDirect0( JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -169,11 +169,11 @@ void Java_org_rocksdb_RocksIterator_seekForPrevDirect0( } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: seekForPrev0 * Signature: (J[BI)V */ -void Java_org_rocksdb_RocksIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_RocksIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_len) { @@ -189,11 +189,11 @@ void Java_org_rocksdb_RocksIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/, * the Java wrapper extracts the byte[] and passes it here. * In this case, the buffer offset of the key may be non-zero. * - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: seek0 * Signature: (J[BII)V */ -void Java_org_rocksdb_RocksIterator_seekForPrevByteArray0( +void Java_org_forstdb_RocksIterator_seekForPrevByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -205,11 +205,11 @@ void Java_org_rocksdb_RocksIterator_seekForPrevByteArray0( } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: status0 * Signature: (J)V */ -void Java_org_rocksdb_RocksIterator_status0(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_RocksIterator_status0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); ROCKSDB_NAMESPACE::Status s = it->status(); @@ -222,11 +222,11 @@ void Java_org_rocksdb_RocksIterator_status0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: key0 * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_RocksIterator_key0(JNIEnv* env, jobject /*jobj*/, +jbyteArray Java_org_forstdb_RocksIterator_key0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); ROCKSDB_NAMESPACE::Slice key_slice = it->key(); @@ -243,11 +243,11 @@ jbyteArray Java_org_rocksdb_RocksIterator_key0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: keyDirect0 * Signature: (JLjava/nio/ByteBuffer;II)I */ -jint Java_org_rocksdb_RocksIterator_keyDirect0(JNIEnv* env, jobject /*jobj*/, +jint Java_org_forstdb_RocksIterator_keyDirect0(JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { @@ -261,11 +261,11 @@ jint Java_org_rocksdb_RocksIterator_keyDirect0(JNIEnv* env, jobject /*jobj*/, * This method supports fetching into indirect byte buffers; * the Java wrapper extracts the byte[] and passes it here. * - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: keyByteArray0 * Signature: (J[BII)I */ -jint Java_org_rocksdb_RocksIterator_keyByteArray0(JNIEnv* env, jobject /*jobj*/, +jint Java_org_forstdb_RocksIterator_keyByteArray0(JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jkey, jint jkey_off, jint jkey_len) { @@ -281,11 +281,11 @@ jint Java_org_rocksdb_RocksIterator_keyByteArray0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: value0 * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_RocksIterator_value0(JNIEnv* env, jobject /*jobj*/, +jbyteArray Java_org_forstdb_RocksIterator_value0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); ROCKSDB_NAMESPACE::Slice value_slice = it->value(); @@ -303,11 +303,11 @@ jbyteArray Java_org_rocksdb_RocksIterator_value0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: valueDirect0 * Signature: (JLjava/nio/ByteBuffer;II)I */ -jint Java_org_rocksdb_RocksIterator_valueDirect0(JNIEnv* env, jobject /*jobj*/, +jint Java_org_forstdb_RocksIterator_valueDirect0(JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { @@ -321,11 +321,11 @@ jint Java_org_rocksdb_RocksIterator_valueDirect0(JNIEnv* env, jobject /*jobj*/, * This method supports fetching into indirect byte buffers; * the Java wrapper extracts the byte[] and passes it here. * - * Class: org_rocksdb_RocksIterator + * Class: org_forstdb_RocksIterator * Method: valueByteArray0 * Signature: (J[BII)I */ -jint Java_org_rocksdb_RocksIterator_valueByteArray0( +jint Java_org_forstdb_RocksIterator_valueByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jvalue_target, jint jvalue_off, jint jvalue_len) { auto* it = reinterpret_cast(handle); diff --git a/java/forstjni/jni_perf_context.cc b/java/forstjni/jni_perf_context.cc new file mode 100644 index 000000000..813a3aed7 --- /dev/null +++ b/java/forstjni/jni_perf_context.cc @@ -0,0 +1,1188 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "include/org_forstdb_PerfContext.h" +#include "rocksdb/db.h" +#include "rocksdb/perf_context.h" + +void Java_org_forstdb_PerfContext_reset(JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + perf_context->Reset(); +} + +/* + * Class: org_forstdb_PerfContext + * Method: getUserKeyComparisonCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getUserKeyComparisonCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->user_key_comparison_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheHitCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockCacheHitCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_cache_hit_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockReadCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockReadCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_read_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheIndexHitCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockCacheIndexHitCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_cache_index_hit_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheStandaloneHandleCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockCacheStandaloneHandleCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_cache_standalone_handle_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheRealHandleCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockCacheRealHandleCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_cache_real_handle_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getIndexBlockReadCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getIndexBlockReadCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->index_block_read_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheFilterHitCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockCacheFilterHitCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_cache_filter_hit_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getFilterBlockReadCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getFilterBlockReadCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->filter_block_read_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressionDictBlockReadCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getCompressionDictBlockReadCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->compression_dict_block_read_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockReadByte + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockReadByte(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_read_byte; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockReadTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockReadTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_read_time; +} + +jlong Java_org_forstdb_PerfContext_getBlockReadCpuTime(JNIEnv*, jobject, + jlong jpc_handler) { + // reinterpret_cast(jcf_handle); + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handler); + return perf_context->block_read_cpu_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSecondaryCacheHitCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSecondaryCacheHitCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->secondary_cache_hit_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressedSecCacheInsertRealCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getCompressedSecCacheInsertRealCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->compressed_sec_cache_insert_real_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressedSecCacheInsertDummyCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getCompressedSecCacheInsertDummyCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->compressed_sec_cache_insert_dummy_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressedSecCacheUncompressedBytes + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getCompressedSecCacheUncompressedBytes( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->compressed_sec_cache_uncompressed_bytes; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressedSecCacheCompressedBytes + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getCompressedSecCacheCompressedBytes( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->compressed_sec_cache_compressed_bytes; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockChecksumTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockChecksumTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_checksum_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockDecompressTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockDecompressTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_decompress_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getReadBytes + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getReadBytes(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->get_read_bytes; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getMultigetReadBytes + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getMultigetReadBytes(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->multiget_read_bytes; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getIterReadBytes + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getIterReadBytes(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->iter_read_bytes; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobCacheHitCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlobCacheHitCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->blob_cache_hit_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobReadCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlobReadCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->blob_read_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobReadByte + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlobReadByte(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->blob_read_byte; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobReadTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlobReadTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->blob_read_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobChecksumTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlobChecksumTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->blob_checksum_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobDecompressTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlobDecompressTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->blob_decompress_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getInternal_key_skipped_count + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getInternalKeySkippedCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->internal_key_skipped_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalDeleteSkippedCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getInternalDeleteSkippedCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->internal_delete_skipped_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalRecentSkippedCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getInternalRecentSkippedCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->internal_recent_skipped_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalMergeCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getInternalMergeCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->internal_merge_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalMergePointLookupCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getInternalMergePointLookupCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->internal_merge_point_lookup_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalRangeDelReseekCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getInternalRangeDelReseekCount( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->internal_range_del_reseek_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSnapshotTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSnapshotTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->get_snapshot_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getFromMemtableTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getFromMemtableTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->get_from_memtable_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getFromMemtableCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getFromMemtableCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->get_from_memtable_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getPostProcessTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getPostProcessTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->get_post_process_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getFromOutputFilesTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getFromOutputFilesTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->get_from_output_files_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekOnMemtableTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSeekOnMemtableTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->seek_on_memtable_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekOnMemtableCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSeekOnMemtableCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->seek_on_memtable_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getNextOnMemtableCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getNextOnMemtableCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->next_on_memtable_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getPrevOnMemtableCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getPrevOnMemtableCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->prev_on_memtable_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekChildSeekTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSeekChildSeekTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->seek_child_seek_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekChildSeekCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSeekChildSeekCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->seek_child_seek_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekMinHeapTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSeekMinHeapTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->seek_min_heap_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekMaxHeapTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSeekMaxHeapTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->seek_max_heap_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekInternalSeekTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getSeekInternalSeekTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->seek_internal_seek_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getFindNextUserEntryTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getFindNextUserEntryTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->find_next_user_entry_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteWalTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getWriteWalTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->write_wal_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteMemtableTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getWriteMemtableTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->write_memtable_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteDelayTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getWriteDelayTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->write_delay_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteSchedulingFlushesCompactionsTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getWriteSchedulingFlushesCompactionsTime( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->write_scheduling_flushes_compactions_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getWritePreAndPostProcessTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getWritePreAndPostProcessTime( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->write_pre_and_post_process_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteThreadWaitNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getWriteThreadWaitNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->write_thread_wait_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getDbMutexLockNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getDbMutexLockNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->db_mutex_lock_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getDbConditionWaitNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getDbConditionWaitNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->db_condition_wait_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getMergeOperatorTimeNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getMergeOperatorTimeNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->merge_operator_time_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getReadIndexBlockNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getReadIndexBlockNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->read_index_block_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getReadFilterBlockNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getReadFilterBlockNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->read_filter_block_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getNewTableBlockIterNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getNewTableBlockIterNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->new_table_block_iter_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getNewTableIteratorNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getNewTableIteratorNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->new_table_iterator_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockSeekNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBlockSeekNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->block_seek_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getFindTableNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getFindTableNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->find_table_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBloomMemtableHitCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBloomMemtableHitCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->bloom_memtable_hit_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBloomMemtableMissCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBloomMemtableMissCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->bloom_memtable_miss_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBloomSstHitCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBloomSstHitCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->bloom_sst_hit_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getBloomSstMissCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getBloomSstMissCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->bloom_sst_miss_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getKeyLockWaitTime + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getKeyLockWaitTime(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->key_lock_wait_time; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getKeyLockWaitCount + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getKeyLockWaitCount(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->key_lock_wait_count; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewSequentialFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvNewSequentialFileNanos( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_new_sequential_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewRandomAccessFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvNewRandomAccessFileNanos( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_new_random_access_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewWritableFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvNewWritableFileNanos( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_new_writable_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvReuseWritableFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvReuseWritableFileNanos( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_reuse_writable_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewRandomRwFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvNewRandomRwFileNanos( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_new_random_rw_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewDirectoryNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvNewDirectoryNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_new_directory_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvFileExistsNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvFileExistsNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_file_exists_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvGetChildrenNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvGetChildrenNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_get_children_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvGetChildrenFileAttributesNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvGetChildrenFileAttributesNanos( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_get_children_file_attributes_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvDeleteFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvDeleteFileNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_delete_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvCreateDirNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvCreateDirNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_create_dir_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvCreateDirIfMissingNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvCreateDirIfMissingNanos( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_create_dir_if_missing_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvDeleteDirNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvDeleteDirNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_delete_dir_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvGetFileSizeNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvGetFileSizeNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_get_file_size_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvGetFileModificationTimeNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvGetFileModificationTimeNanos( + JNIEnv*, jobject, jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_get_file_modification_time_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvRenameFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvRenameFileNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_rename_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvLinkFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvLinkFileNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_link_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvLockFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvLockFileNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_lock_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvUnlockFileNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvUnlockFileNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_unlock_file_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewLoggerNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEnvNewLoggerNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->env_new_logger_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getCpuNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getGetCpuNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->get_cpu_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getIterNextCpuNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getIterNextCpuNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->iter_next_cpu_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getIterPrevCpuNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getIterPrevCpuNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->iter_prev_cpu_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getIterSeekCpuNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getIterSeekCpuNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->iter_seek_cpu_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getEncryptDataNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getEncryptDataNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->encrypt_data_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getDecryptDataNanos + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getDecryptDataNanos(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->decrypt_data_nanos; +} + +/* + * Class: org_forstdb_PerfContext + * Method: getNumberAsyncSeek + * Signature: (J)J + */ +jlong Java_org_forstdb_PerfContext_getNumberAsyncSeek(JNIEnv*, jobject, + jlong jpc_handle) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = + reinterpret_cast(jpc_handle); + return perf_context->number_async_seek; +} diff --git a/java/rocksjni/jnicallback.cc b/java/forstjni/jnicallback.cc similarity index 92% rename from java/rocksjni/jnicallback.cc rename to java/forstjni/jnicallback.cc index f2742cd88..29db5b960 100644 --- a/java/rocksjni/jnicallback.cc +++ b/java/forstjni/jnicallback.cc @@ -4,13 +4,13 @@ // (found in the LICENSE.Apache file in the root directory). // // This file implements the callback "bridge" between Java and C++ for -// JNI Callbacks from C++ to sub-classes or org.rocksdb.RocksCallbackObject +// JNI Callbacks from C++ to sub-classes or org.forstdb.RocksCallbackObject -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" #include -#include "rocksjni/portal.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { JniCallback::JniCallback(JNIEnv* env, jobject jcallback_obj) { diff --git a/java/rocksjni/jnicallback.h b/java/forstjni/jnicallback.h similarity index 100% rename from java/rocksjni/jnicallback.h rename to java/forstjni/jnicallback.h diff --git a/java/forstjni/kv_helper.h b/java/forstjni/kv_helper.h new file mode 100644 index 000000000..4caffa16c --- /dev/null +++ b/java/forstjni/kv_helper.h @@ -0,0 +1,284 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file defines helper methods for Java API write methods +// + +#pragma once + +#include + +#include +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "forstjni/portal.h" + +namespace ROCKSDB_NAMESPACE { + +/** + * @brief Exception class used to make the flow of key/value (Put(), Get(), + * Merge(), ...) calls clearer. + * + * This class is used by Java API JNI methods in try { save/fetch } catch { ... + * } style. + * + */ +class KVException : public std::exception { + public: + // These values are expected on Java API calls to represent the result of a + // Get() which has failed; a negative length is returned to indicate an error. + static const int kNotFound = -1; // the key was not found in RocksDB + static const int kStatusError = + -2; // there was some other error fetching the value for the key + + /** + * @brief Throw a KVException (and potentially a Java exception) if the + * RocksDB status is "bad" + * + * @param env JNI environment needed to create a Java exception + * @param status RocksDB status to examine + */ + static void ThrowOnError(JNIEnv* env, const Status& status) { + if (status.ok()) { + return; + } + if (status.IsNotFound()) { + // IsNotFound does not generate a Java Exception, any other bad status + // does.. + throw KVException(kNotFound); + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + throw KVException(kStatusError); + } + + /** + * @brief Throw a KVException and a Java exception + * + * @param env JNI environment needed to create a Java exception + * @param message content of the exception we will throw + */ + static void ThrowNew(JNIEnv* env, const std::string& message) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, message); + throw KVException(kStatusError); + } + + /** + * @brief Throw a KVException if there is already a Java exception in the JNI + * enviroment + * + * @param env + */ + static void ThrowOnError(JNIEnv* env) { + if (env->ExceptionCheck()) { + throw KVException(kStatusError); + } + } + + KVException(jint code) : kCode_(code){}; + + virtual const char* what() const throw() { + return "Exception raised by JNI. There may be a Java exception in the " + "JNIEnv. Please check!"; + } + + jint Code() const { return kCode_; } + + private: + jint kCode_; +}; + +/** + * @brief Construct a slice with the contents of a Java byte array + * + * The slice refers to an array into which the Java byte array's whole region is + * copied + */ +class JByteArraySlice { + public: + JByteArraySlice(JNIEnv* env, const jbyteArray& jarr, const jint jarr_off, + const jint jarr_len) + : arr_(new jbyte[jarr_len]), + slice_(reinterpret_cast(arr_), jarr_len) { + env->GetByteArrayRegion(jarr, jarr_off, jarr_len, arr_); + KVException::ThrowOnError(env); + }; + + ~JByteArraySlice() { + slice_.clear(); + delete[] arr_; + }; + + Slice& slice() { return slice_; } + + private: + jbyte* arr_; + Slice slice_; +}; + +/** + * @brief Construct a slice with the contents of a direct Java ByterBuffer + * + * The slice refers directly to the contents of the buffer, no copy is made. + * + */ +class JDirectBufferSlice { + public: + JDirectBufferSlice(JNIEnv* env, const jobject& jbuffer, + const jint jbuffer_off, const jint jbuffer_len) + : slice_(static_cast(env->GetDirectBufferAddress(jbuffer)) + + jbuffer_off, + jbuffer_len) { + KVException::ThrowOnError(env); + jlong capacity = env->GetDirectBufferCapacity(jbuffer); + if (capacity < jbuffer_off + jbuffer_len) { + auto message = "Direct buffer offset " + std::to_string(jbuffer_off) + + " + length " + std::to_string(jbuffer_len) + + " exceeds capacity " + std::to_string(capacity); + KVException::ThrowNew(env, message); + slice_.clear(); + } + } + + ~JDirectBufferSlice() { slice_.clear(); }; + + Slice& slice() { return slice_; } + + private: + Slice slice_; +}; + +/** + * @brief Wrap a pinnable slice with a method to retrieve the contents back into + * Java + * + * The Java Byte Array version sets the byte array's region from the slice + */ +class JByteArrayPinnableSlice { + public: + /** + * @brief Construct a new JByteArrayPinnableSlice object referring to an + * existing java byte buffer + * + * @param env + * @param jbuffer + * @param jbuffer_off + * @param jbuffer_len + */ + JByteArrayPinnableSlice(JNIEnv* env, const jbyteArray& jbuffer, + const jint jbuffer_off, const jint jbuffer_len) + : env_(env), + jbuffer_(jbuffer), + jbuffer_off_(jbuffer_off), + jbuffer_len_(jbuffer_len){}; + + /** + * @brief Construct an empty new JByteArrayPinnableSlice object + * + */ + JByteArrayPinnableSlice(JNIEnv* env) : env_(env){}; + + PinnableSlice& pinnable_slice() { return pinnable_slice_; } + + ~JByteArrayPinnableSlice() { pinnable_slice_.Reset(); }; + + /** + * @brief copy back contents of the pinnable slice into the Java ByteBuffer + * + * @return jint min of size of buffer and number of bytes in value for + * requested key + */ + jint Fetch() { + const jint pinnable_len = static_cast(pinnable_slice_.size()); + const jint result_len = std::min(jbuffer_len_, pinnable_len); + env_->SetByteArrayRegion( + jbuffer_, jbuffer_off_, result_len, + reinterpret_cast(pinnable_slice_.data())); + KVException::ThrowOnError( + env_); // exception thrown: ArrayIndexOutOfBoundsException + + return pinnable_len; + }; + + /** + * @brief create a new Java buffer and copy the result into it + * + * @return jbyteArray the java buffer holding the result + */ + jbyteArray NewByteArray() { + const jint pinnable_len = static_cast(pinnable_slice_.size()); + jbyteArray jbuffer = env_->NewByteArray(static_cast(pinnable_len)); + KVException::ThrowOnError(env_); // OutOfMemoryError + + env_->SetByteArrayRegion( + jbuffer, 0, pinnable_len, + reinterpret_cast(pinnable_slice_.data())); + KVException::ThrowOnError(env_); // ArrayIndexOutOfBoundsException + + return jbuffer; + } + + private: + JNIEnv* env_; + jbyteArray jbuffer_; + jint jbuffer_off_; + jint jbuffer_len_; + PinnableSlice pinnable_slice_; +}; + +/** + * @brief Wrap a pinnable slice with a method to retrieve the contents back into + * Java + * + * The Java Direct Buffer version copies the memory of the buffer from the slice + */ +class JDirectBufferPinnableSlice { + public: + JDirectBufferPinnableSlice(JNIEnv* env, const jobject& jbuffer, + const jint jbuffer_off, const jint jbuffer_len) + : buffer_(static_cast(env->GetDirectBufferAddress(jbuffer)) + + jbuffer_off), + jbuffer_len_(jbuffer_len) { + jlong capacity = env->GetDirectBufferCapacity(jbuffer); + if (capacity < jbuffer_off + jbuffer_len) { + auto message = + "Invalid value argument. Capacity is less than requested region. " + "offset " + + std::to_string(jbuffer_off) + " + length " + + std::to_string(jbuffer_len) + " exceeds capacity " + + std::to_string(capacity); + KVException::ThrowNew(env, message); + } + } + + PinnableSlice& pinnable_slice() { return pinnable_slice_; } + + ~JDirectBufferPinnableSlice() { pinnable_slice_.Reset(); }; + + /** + * @brief copy back contents of the pinnable slice into the Java DirectBuffer + * + * @return jint min of size of buffer and number of bytes in value for + * requested key + */ + jint Fetch() { + const jint pinnable_len = static_cast(pinnable_slice_.size()); + const jint result_len = std::min(jbuffer_len_, pinnable_len); + + memcpy(buffer_, pinnable_slice_.data(), result_len); + return pinnable_len; + }; + + private: + char* buffer_; + jint jbuffer_len_; + PinnableSlice pinnable_slice_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/java/rocksjni/loggerjnicallback.cc b/java/forstjni/loggerjnicallback.cc similarity index 92% rename from java/rocksjni/loggerjnicallback.cc rename to java/forstjni/loggerjnicallback.cc index aa9f95cd4..82724e945 100644 --- a/java/rocksjni/loggerjnicallback.cc +++ b/java/forstjni/loggerjnicallback.cc @@ -6,14 +6,14 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::Logger. -#include "rocksjni/loggerjnicallback.h" +#include "forstjni/loggerjnicallback.h" #include #include -#include "include/org_rocksdb_Logger.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_Logger.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { @@ -223,11 +223,11 @@ LoggerJniCallback::~LoggerJniCallback() { } // namespace ROCKSDB_NAMESPACE /* - * Class: org_rocksdb_Logger + * Class: org_forstdb_Logger * Method: createNewLoggerOptions * Signature: (J)J */ -jlong Java_org_rocksdb_Logger_createNewLoggerOptions(JNIEnv* env, jobject jobj, +jlong Java_org_forstdb_Logger_createNewLoggerOptions(JNIEnv* env, jobject jobj, jlong joptions) { auto* sptr_logger = new std::shared_ptr( new ROCKSDB_NAMESPACE::LoggerJniCallback(env, jobj)); @@ -240,11 +240,11 @@ jlong Java_org_rocksdb_Logger_createNewLoggerOptions(JNIEnv* env, jobject jobj, } /* - * Class: org_rocksdb_Logger + * Class: org_forstdb_Logger * Method: createNewLoggerDbOptions * Signature: (J)J */ -jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(JNIEnv* env, +jlong Java_org_forstdb_Logger_createNewLoggerDbOptions(JNIEnv* env, jobject jobj, jlong jdb_options) { auto* sptr_logger = new std::shared_ptr( @@ -259,11 +259,11 @@ jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(JNIEnv* env, } /* - * Class: org_rocksdb_Logger + * Class: org_forstdb_Logger * Method: setInfoLogLevel * Signature: (JB)V */ -void Java_org_rocksdb_Logger_setInfoLogLevel(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_Logger_setInfoLogLevel(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jbyte jlog_level) { auto* handle = reinterpret_cast*>( @@ -273,11 +273,11 @@ void Java_org_rocksdb_Logger_setInfoLogLevel(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_Logger + * Class: org_forstdb_Logger * Method: infoLogLevel * Signature: (J)B */ -jbyte Java_org_rocksdb_Logger_infoLogLevel(JNIEnv* /*env*/, jobject /*jobj*/, +jbyte Java_org_forstdb_Logger_infoLogLevel(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* handle = reinterpret_cast*>( @@ -286,11 +286,11 @@ jbyte Java_org_rocksdb_Logger_infoLogLevel(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_Logger + * Class: org_forstdb_Logger * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_Logger_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_Logger_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* handle = reinterpret_cast*>( diff --git a/java/rocksjni/loggerjnicallback.h b/java/forstjni/loggerjnicallback.h similarity index 97% rename from java/rocksjni/loggerjnicallback.h rename to java/forstjni/loggerjnicallback.h index 57774988c..ec1393aa3 100644 --- a/java/rocksjni/loggerjnicallback.h +++ b/java/forstjni/loggerjnicallback.h @@ -16,7 +16,7 @@ #include "port/port.h" #include "rocksdb/env.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/lru_cache.cc b/java/forstjni/lru_cache.cc similarity index 78% rename from java/rocksjni/lru_cache.cc rename to java/forstjni/lru_cache.cc index 56dffa2f0..cc2bb1851 100644 --- a/java/rocksjni/lru_cache.cc +++ b/java/forstjni/lru_cache.cc @@ -10,15 +10,15 @@ #include -#include "include/org_rocksdb_LRUCache.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "include/org_forstdb_LRUCache.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_LRUCache + * Class: org_forstdb_LRUCache * Method: newLRUCache * Signature: (JIZD)J */ -jlong Java_org_rocksdb_LRUCache_newLRUCache(JNIEnv* /*env*/, jclass /*jcls*/, +jlong Java_org_forstdb_LRUCache_newLRUCache(JNIEnv* /*env*/, jclass /*jcls*/, jlong jcapacity, jint jnum_shard_bits, jboolean jstrict_capacity_limit, @@ -29,18 +29,18 @@ jlong Java_org_rocksdb_LRUCache_newLRUCache(JNIEnv* /*env*/, jclass /*jcls*/, static_cast(jcapacity), static_cast(jnum_shard_bits), static_cast(jstrict_capacity_limit), static_cast(jhigh_pri_pool_ratio), - nullptr /* memory_allocator */, rocksdb::kDefaultToAdaptiveMutex, - rocksdb::kDefaultCacheMetadataChargePolicy, + nullptr /* memory_allocator */, ROCKSDB_NAMESPACE::kDefaultToAdaptiveMutex, + ROCKSDB_NAMESPACE::kDefaultCacheMetadataChargePolicy, static_cast(jlow_pri_pool_ratio))); return GET_CPLUSPLUS_POINTER(sptr_lru_cache); } /* - * Class: org_rocksdb_LRUCache + * Class: org_forstdb_LRUCache * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_LRUCache_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_LRUCache_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_lru_cache = diff --git a/java/rocksjni/memory_util.cc b/java/forstjni/memory_util.cc similarity index 95% rename from java/rocksjni/memory_util.cc rename to java/forstjni/memory_util.cc index c87c4f403..9a40b6d75 100644 --- a/java/rocksjni/memory_util.cc +++ b/java/forstjni/memory_util.cc @@ -12,15 +12,15 @@ #include #include -#include "include/org_rocksdb_MemoryUtil.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_MemoryUtil.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_MemoryUtil + * Class: org_forstdb_MemoryUtil * Method: getApproximateMemoryUsageByType * Signature: ([J[J)Ljava/util/Map; */ -jobject Java_org_rocksdb_MemoryUtil_getApproximateMemoryUsageByType( +jobject Java_org_forstdb_MemoryUtil_getApproximateMemoryUsageByType( JNIEnv *env, jclass, jlongArray jdb_handles, jlongArray jcache_handles) { jboolean has_exception = JNI_FALSE; std::vector dbs = diff --git a/java/rocksjni/memtablejni.cc b/java/forstjni/memtablejni.cc similarity index 79% rename from java/rocksjni/memtablejni.cc rename to java/forstjni/memtablejni.cc index a4d02f354..59473c7c1 100644 --- a/java/rocksjni/memtablejni.cc +++ b/java/forstjni/memtablejni.cc @@ -5,20 +5,20 @@ // // This file implements the "bridge" between Java and C++ for MemTables. -#include "include/org_rocksdb_HashLinkedListMemTableConfig.h" -#include "include/org_rocksdb_HashSkipListMemTableConfig.h" -#include "include/org_rocksdb_SkipListMemTableConfig.h" -#include "include/org_rocksdb_VectorMemTableConfig.h" +#include "include/org_forstdb_HashLinkedListMemTableConfig.h" +#include "include/org_forstdb_HashSkipListMemTableConfig.h" +#include "include/org_forstdb_SkipListMemTableConfig.h" +#include "include/org_forstdb_VectorMemTableConfig.h" #include "rocksdb/memtablerep.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_HashSkipListMemTableConfig + * Class: org_forstdb_HashSkipListMemTableConfig * Method: newMemTableFactoryHandle * Signature: (JII)J */ -jlong Java_org_rocksdb_HashSkipListMemTableConfig_newMemTableFactoryHandle( +jlong Java_org_forstdb_HashSkipListMemTableConfig_newMemTableFactoryHandle( JNIEnv* env, jobject /*jobj*/, jlong jbucket_count, jint jheight, jint jbranching_factor) { ROCKSDB_NAMESPACE::Status s = @@ -33,11 +33,11 @@ jlong Java_org_rocksdb_HashSkipListMemTableConfig_newMemTableFactoryHandle( } /* - * Class: org_rocksdb_HashLinkedListMemTableConfig + * Class: org_forstdb_HashLinkedListMemTableConfig * Method: newMemTableFactoryHandle * Signature: (JJIZI)J */ -jlong Java_org_rocksdb_HashLinkedListMemTableConfig_newMemTableFactoryHandle( +jlong Java_org_forstdb_HashLinkedListMemTableConfig_newMemTableFactoryHandle( JNIEnv* env, jobject /*jobj*/, jlong jbucket_count, jlong jhuge_page_tlb_size, jint jbucket_entries_logging_threshold, jboolean jif_log_bucket_dist_when_flash, jint jthreshold_use_skiplist) { @@ -60,11 +60,11 @@ jlong Java_org_rocksdb_HashLinkedListMemTableConfig_newMemTableFactoryHandle( } /* - * Class: org_rocksdb_VectorMemTableConfig + * Class: org_forstdb_VectorMemTableConfig * Method: newMemTableFactoryHandle * Signature: (J)J */ -jlong Java_org_rocksdb_VectorMemTableConfig_newMemTableFactoryHandle( +jlong Java_org_forstdb_VectorMemTableConfig_newMemTableFactoryHandle( JNIEnv* env, jobject /*jobj*/, jlong jreserved_size) { ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jreserved_size); @@ -77,11 +77,11 @@ jlong Java_org_rocksdb_VectorMemTableConfig_newMemTableFactoryHandle( } /* - * Class: org_rocksdb_SkipListMemTableConfig + * Class: org_forstdb_SkipListMemTableConfig * Method: newMemTableFactoryHandle0 * Signature: (J)J */ -jlong Java_org_rocksdb_SkipListMemTableConfig_newMemTableFactoryHandle0( +jlong Java_org_forstdb_SkipListMemTableConfig_newMemTableFactoryHandle0( JNIEnv* env, jobject /*jobj*/, jlong jlookahead) { ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jlookahead); diff --git a/java/rocksjni/merge_operator.cc b/java/forstjni/merge_operator.cc similarity index 80% rename from java/rocksjni/merge_operator.cc rename to java/forstjni/merge_operator.cc index ce3c5df56..e5da11fb9 100644 --- a/java/rocksjni/merge_operator.cc +++ b/java/forstjni/merge_operator.cc @@ -16,24 +16,24 @@ #include #include -#include "include/org_rocksdb_StringAppendOperator.h" -#include "include/org_rocksdb_UInt64AddOperator.h" +#include "include/org_forstdb_StringAppendOperator.h" +#include "include/org_forstdb_UInt64AddOperator.h" #include "rocksdb/db.h" #include "rocksdb/memtablerep.h" #include "rocksdb/options.h" #include "rocksdb/slice_transform.h" #include "rocksdb/statistics.h" #include "rocksdb/table.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" #include "utilities/merge_operators.h" /* - * Class: org_rocksdb_StringAppendOperator + * Class: org_forstdb_StringAppendOperator * Method: newSharedStringAppendOperator * Signature: (C)J */ -jlong Java_org_rocksdb_StringAppendOperator_newSharedStringAppendOperator__C( +jlong Java_org_forstdb_StringAppendOperator_newSharedStringAppendOperator__C( JNIEnv* /*env*/, jclass /*jclazz*/, jchar jdelim) { auto* sptr_string_append_op = new std::shared_ptr( @@ -42,7 +42,7 @@ jlong Java_org_rocksdb_StringAppendOperator_newSharedStringAppendOperator__C( return GET_CPLUSPLUS_POINTER(sptr_string_append_op); } -jlong Java_org_rocksdb_StringAppendOperator_newSharedStringAppendOperator__Ljava_lang_String_2( +jlong Java_org_forstdb_StringAppendOperator_newSharedStringAppendOperator__Ljava_lang_String_2( JNIEnv* env, jclass /*jclass*/, jstring jdelim) { jboolean has_exception = JNI_FALSE; auto delim = @@ -57,11 +57,11 @@ jlong Java_org_rocksdb_StringAppendOperator_newSharedStringAppendOperator__Ljava } /* - * Class: org_rocksdb_StringAppendOperator + * Class: org_forstdb_StringAppendOperator * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_StringAppendOperator_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_StringAppendOperator_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_string_append_op = @@ -71,11 +71,11 @@ void Java_org_rocksdb_StringAppendOperator_disposeInternal(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_UInt64AddOperator + * Class: org_forstdb_UInt64AddOperator * Method: newSharedUInt64AddOperator * Signature: ()J */ -jlong Java_org_rocksdb_UInt64AddOperator_newSharedUInt64AddOperator( +jlong Java_org_forstdb_UInt64AddOperator_newSharedUInt64AddOperator( JNIEnv* /*env*/, jclass /*jclazz*/) { auto* sptr_uint64_add_op = new std::shared_ptr( @@ -84,11 +84,11 @@ jlong Java_org_rocksdb_UInt64AddOperator_newSharedUInt64AddOperator( } /* - * Class: org_rocksdb_UInt64AddOperator + * Class: org_forstdb_UInt64AddOperator * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_UInt64AddOperator_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_UInt64AddOperator_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_uint64_add_op = diff --git a/java/rocksjni/native_comparator_wrapper_test.cc b/java/forstjni/native_comparator_wrapper_test.cc similarity index 83% rename from java/rocksjni/native_comparator_wrapper_test.cc rename to java/forstjni/native_comparator_wrapper_test.cc index ac33ca22d..708d6fd4c 100644 --- a/java/rocksjni/native_comparator_wrapper_test.cc +++ b/java/forstjni/native_comparator_wrapper_test.cc @@ -7,10 +7,10 @@ #include -#include "include/org_rocksdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper.h" +#include "include/org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper.h" #include "rocksdb/comparator.h" #include "rocksdb/slice.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" namespace ROCKSDB_NAMESPACE { @@ -33,11 +33,11 @@ class NativeComparatorWrapperTestStringComparator : public Comparator { } // namespace ROCKSDB_NAMESPACE /* - * Class: org_rocksdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper + * Class: org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper * Method: newStringComparator * Signature: ()J */ -jlong Java_org_rocksdb_NativeComparatorWrapperTest_00024NativeStringComparatorWrapper_newStringComparator( +jlong Java_org_forstdb_NativeComparatorWrapperTest_00024NativeStringComparatorWrapper_newStringComparator( JNIEnv* /*env*/, jobject /*jobj*/) { auto* comparator = new ROCKSDB_NAMESPACE::NativeComparatorWrapperTestStringComparator(); diff --git a/java/rocksjni/optimistic_transaction_db.cc b/java/forstjni/optimistic_transaction_db.cc similarity index 88% rename from java/rocksjni/optimistic_transaction_db.cc rename to java/forstjni/optimistic_transaction_db.cc index 238224f58..0e6fcf1c7 100644 --- a/java/rocksjni/optimistic_transaction_db.cc +++ b/java/forstjni/optimistic_transaction_db.cc @@ -10,18 +10,18 @@ #include -#include "include/org_rocksdb_OptimisticTransactionDB.h" +#include "include/org_forstdb_OptimisticTransactionDB.h" #include "rocksdb/options.h" #include "rocksdb/utilities/transaction.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: open * Signature: (JLjava/lang/String;)J */ -jlong Java_org_rocksdb_OptimisticTransactionDB_open__JLjava_lang_String_2( +jlong Java_org_forstdb_OptimisticTransactionDB_open__JLjava_lang_String_2( JNIEnv* env, jclass, jlong joptions_handle, jstring jdb_path) { const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); if (db_path == nullptr) { @@ -46,12 +46,12 @@ jlong Java_org_rocksdb_OptimisticTransactionDB_open__JLjava_lang_String_2( } /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: open * Signature: (JLjava/lang/String;[[B[J)[J */ jlongArray -Java_org_rocksdb_OptimisticTransactionDB_open__JLjava_lang_String_2_3_3B_3J( +Java_org_forstdb_OptimisticTransactionDB_open__JLjava_lang_String_2_3_3B_3J( JNIEnv* env, jclass, jlong jdb_options_handle, jstring jdb_path, jobjectArray jcolumn_names, jlongArray jcolumn_options_handles) { const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); @@ -141,11 +141,11 @@ Java_org_rocksdb_OptimisticTransactionDB_open__JLjava_lang_String_2_3_3B_3J( } /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_OptimisticTransactionDB_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_OptimisticTransactionDB_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* optimistic_txn_db = reinterpret_cast(jhandle); @@ -154,11 +154,11 @@ void Java_org_rocksdb_OptimisticTransactionDB_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: closeDatabase * Signature: (J)V */ -void Java_org_rocksdb_OptimisticTransactionDB_closeDatabase(JNIEnv* env, jclass, +void Java_org_forstdb_OptimisticTransactionDB_closeDatabase(JNIEnv* env, jclass, jlong jhandle) { auto* optimistic_txn_db = reinterpret_cast(jhandle); @@ -168,11 +168,11 @@ void Java_org_rocksdb_OptimisticTransactionDB_closeDatabase(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: beginTransaction * Signature: (JJ)J */ -jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction__JJ( +jlong Java_org_forstdb_OptimisticTransactionDB_beginTransaction__JJ( JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle) { auto* optimistic_txn_db = reinterpret_cast(jhandle); @@ -184,11 +184,11 @@ jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction__JJ( } /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: beginTransaction * Signature: (JJJ)J */ -jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction__JJJ( +jlong Java_org_forstdb_OptimisticTransactionDB_beginTransaction__JJJ( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jwrite_options_handle, jlong joptimistic_txn_options_handle) { auto* optimistic_txn_db = @@ -204,11 +204,11 @@ jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction__JJJ( } /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: beginTransaction_withOld * Signature: (JJJ)J */ -jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJ( +jlong Java_org_forstdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJ( JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, jlong jold_txn_handle) { auto* optimistic_txn_db = @@ -230,11 +230,11 @@ jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJ( } /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: beginTransaction_withOld * Signature: (JJJJ)J */ -jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJJ( +jlong Java_org_forstdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJJ( JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, jlong joptimistic_txn_options_handle, jlong jold_txn_handle) { auto* optimistic_txn_db = @@ -258,11 +258,11 @@ jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJJ( } /* - * Class: org_rocksdb_OptimisticTransactionDB + * Class: org_forstdb_OptimisticTransactionDB * Method: getBaseDB * Signature: (J)J */ -jlong Java_org_rocksdb_OptimisticTransactionDB_getBaseDB(JNIEnv*, jobject, +jlong Java_org_forstdb_OptimisticTransactionDB_getBaseDB(JNIEnv*, jobject, jlong jhandle) { auto* optimistic_txn_db = reinterpret_cast(jhandle); diff --git a/java/rocksjni/optimistic_transaction_options.cc b/java/forstjni/optimistic_transaction_options.cc similarity index 72% rename from java/rocksjni/optimistic_transaction_options.cc rename to java/forstjni/optimistic_transaction_options.cc index 501c6c4fb..feb5e0238 100644 --- a/java/rocksjni/optimistic_transaction_options.cc +++ b/java/forstjni/optimistic_transaction_options.cc @@ -8,17 +8,17 @@ #include -#include "include/org_rocksdb_OptimisticTransactionOptions.h" +#include "include/org_forstdb_OptimisticTransactionOptions.h" #include "rocksdb/comparator.h" #include "rocksdb/utilities/optimistic_transaction_db.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_OptimisticTransactionOptions + * Class: org_forstdb_OptimisticTransactionOptions * Method: newOptimisticTransactionOptions * Signature: ()J */ -jlong Java_org_rocksdb_OptimisticTransactionOptions_newOptimisticTransactionOptions( +jlong Java_org_forstdb_OptimisticTransactionOptions_newOptimisticTransactionOptions( JNIEnv* /*env*/, jclass /*jcls*/) { ROCKSDB_NAMESPACE::OptimisticTransactionOptions* opts = new ROCKSDB_NAMESPACE::OptimisticTransactionOptions(); @@ -26,11 +26,11 @@ jlong Java_org_rocksdb_OptimisticTransactionOptions_newOptimisticTransactionOpti } /* - * Class: org_rocksdb_OptimisticTransactionOptions + * Class: org_forstdb_OptimisticTransactionOptions * Method: isSetSnapshot * Signature: (J)Z */ -jboolean Java_org_rocksdb_OptimisticTransactionOptions_isSetSnapshot( +jboolean Java_org_forstdb_OptimisticTransactionOptions_isSetSnapshot( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = reinterpret_cast( @@ -39,11 +39,11 @@ jboolean Java_org_rocksdb_OptimisticTransactionOptions_isSetSnapshot( } /* - * Class: org_rocksdb_OptimisticTransactionOptions + * Class: org_forstdb_OptimisticTransactionOptions * Method: setSetSnapshot * Signature: (JZ)V */ -void Java_org_rocksdb_OptimisticTransactionOptions_setSetSnapshot( +void Java_org_forstdb_OptimisticTransactionOptions_setSetSnapshot( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean jset_snapshot) { auto* opts = reinterpret_cast( @@ -52,11 +52,11 @@ void Java_org_rocksdb_OptimisticTransactionOptions_setSetSnapshot( } /* - * Class: org_rocksdb_OptimisticTransactionOptions + * Class: org_forstdb_OptimisticTransactionOptions * Method: setComparator * Signature: (JJ)V */ -void Java_org_rocksdb_OptimisticTransactionOptions_setComparator( +void Java_org_forstdb_OptimisticTransactionOptions_setComparator( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jcomparator_handle) { auto* opts = @@ -67,11 +67,11 @@ void Java_org_rocksdb_OptimisticTransactionOptions_setComparator( } /* - * Class: org_rocksdb_OptimisticTransactionOptions + * Class: org_forstdb_OptimisticTransactionOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_OptimisticTransactionOptions_disposeInternal( +void Java_org_forstdb_OptimisticTransactionOptions_disposeInternal( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { delete reinterpret_cast( jhandle); diff --git a/java/rocksjni/options.cc b/java/forstjni/options.cc similarity index 76% rename from java/rocksjni/options.cc rename to java/forstjni/options.cc index 724d298e7..02690f60f 100644 --- a/java/rocksjni/options.cc +++ b/java/forstjni/options.cc @@ -15,13 +15,13 @@ #include #include -#include "include/org_rocksdb_ColumnFamilyOptions.h" -#include "include/org_rocksdb_ComparatorOptions.h" -#include "include/org_rocksdb_DBOptions.h" -#include "include/org_rocksdb_FlushOptions.h" -#include "include/org_rocksdb_Options.h" -#include "include/org_rocksdb_ReadOptions.h" -#include "include/org_rocksdb_WriteOptions.h" +#include "include/org_forstdb_ColumnFamilyOptions.h" +#include "include/org_forstdb_ComparatorOptions.h" +#include "include/org_forstdb_DBOptions.h" +#include "include/org_forstdb_FlushOptions.h" +#include "include/org_forstdb_Options.h" +#include "include/org_forstdb_ReadOptions.h" +#include "include/org_forstdb_WriteOptions.h" #include "rocksdb/comparator.h" #include "rocksdb/convenience.h" #include "rocksdb/db.h" @@ -32,29 +32,29 @@ #include "rocksdb/sst_partitioner.h" #include "rocksdb/statistics.h" #include "rocksdb/table.h" -#include "rocksjni/comparatorjnicallback.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" -#include "rocksjni/statisticsjni.h" -#include "rocksjni/table_filter_jnicallback.h" +#include "forstjni/comparatorjnicallback.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" +#include "forstjni/statisticsjni.h" +#include "forstjni/table_filter_jnicallback.h" #include "utilities/merge_operators.h" /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: newOptions * Signature: ()J */ -jlong Java_org_rocksdb_Options_newOptions__(JNIEnv*, jclass) { +jlong Java_org_forstdb_Options_newOptions__(JNIEnv*, jclass) { auto* op = new ROCKSDB_NAMESPACE::Options(); return GET_CPLUSPLUS_POINTER(op); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: newOptions * Signature: (JJ)J */ -jlong Java_org_rocksdb_Options_newOptions__JJ(JNIEnv*, jclass, jlong jdboptions, +jlong Java_org_forstdb_Options_newOptions__JJ(JNIEnv*, jclass, jlong jdboptions, jlong jcfoptions) { auto* dbOpt = reinterpret_cast(jdboptions); @@ -65,33 +65,33 @@ jlong Java_org_rocksdb_Options_newOptions__JJ(JNIEnv*, jclass, jlong jdboptions, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: copyOptions * Signature: (J)J */ -jlong Java_org_rocksdb_Options_copyOptions(JNIEnv*, jclass, jlong jhandle) { +jlong Java_org_forstdb_Options_copyOptions(JNIEnv*, jclass, jlong jhandle) { auto new_opt = new ROCKSDB_NAMESPACE::Options( *(reinterpret_cast(jhandle))); return GET_CPLUSPLUS_POINTER(new_opt); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_Options_disposeInternal(JNIEnv*, jobject, jlong handle) { +void Java_org_forstdb_Options_disposeInternal(JNIEnv*, jobject, jlong handle) { auto* op = reinterpret_cast(handle); assert(op != nullptr); delete op; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setIncreaseParallelism * Signature: (JI)V */ -void Java_org_rocksdb_Options_setIncreaseParallelism(JNIEnv*, jobject, +void Java_org_forstdb_Options_setIncreaseParallelism(JNIEnv*, jobject, jlong jhandle, jint totalThreads) { reinterpret_cast(jhandle)->IncreaseParallelism( @@ -99,33 +99,33 @@ void Java_org_rocksdb_Options_setIncreaseParallelism(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCreateIfMissing * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setCreateIfMissing(JNIEnv*, jobject, +void Java_org_forstdb_Options_setCreateIfMissing(JNIEnv*, jobject, jlong jhandle, jboolean flag) { reinterpret_cast(jhandle)->create_if_missing = flag; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: createIfMissing * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_createIfMissing(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_createIfMissing(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->create_if_missing; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCreateMissingColumnFamilies * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setCreateMissingColumnFamilies(JNIEnv*, jobject, +void Java_org_forstdb_Options_setCreateMissingColumnFamilies(JNIEnv*, jobject, jlong jhandle, jboolean flag) { reinterpret_cast(jhandle) @@ -133,22 +133,22 @@ void Java_org_rocksdb_Options_setCreateMissingColumnFamilies(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: createMissingColumnFamilies * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_createMissingColumnFamilies(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_createMissingColumnFamilies(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->create_missing_column_families; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setComparatorHandle * Signature: (JI)V */ -void Java_org_rocksdb_Options_setComparatorHandle__JI(JNIEnv*, jobject, +void Java_org_forstdb_Options_setComparatorHandle__JI(JNIEnv*, jobject, jlong jhandle, jint builtinComparator) { switch (builtinComparator) { @@ -164,11 +164,11 @@ void Java_org_rocksdb_Options_setComparatorHandle__JI(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setComparatorHandle * Signature: (JJB)V */ -void Java_org_rocksdb_Options_setComparatorHandle__JJB(JNIEnv*, jobject, +void Java_org_forstdb_Options_setComparatorHandle__JJB(JNIEnv*, jobject, jlong jopt_handle, jlong jcomparator_handle, jbyte jcomparator_type) { @@ -191,11 +191,11 @@ void Java_org_rocksdb_Options_setComparatorHandle__JJB(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMergeOperatorName * Signature: (JJjava/lang/String)V */ -void Java_org_rocksdb_Options_setMergeOperatorName(JNIEnv* env, jobject, +void Java_org_forstdb_Options_setMergeOperatorName(JNIEnv* env, jobject, jlong jhandle, jstring jop_name) { const char* op_name = env->GetStringUTFChars(jop_name, nullptr); @@ -212,11 +212,11 @@ void Java_org_rocksdb_Options_setMergeOperatorName(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMergeOperator * Signature: (JJjava/lang/String)V */ -void Java_org_rocksdb_Options_setMergeOperator(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setMergeOperator(JNIEnv*, jobject, jlong jhandle, jlong mergeOperatorHandle) { reinterpret_cast(jhandle)->merge_operator = *(reinterpret_cast*>( @@ -224,11 +224,11 @@ void Java_org_rocksdb_Options_setMergeOperator(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompactionFilterHandle * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setCompactionFilterHandle( +void Java_org_forstdb_Options_setCompactionFilterHandle( JNIEnv*, jobject, jlong jopt_handle, jlong jcompactionfilter_handle) { reinterpret_cast(jopt_handle) ->compaction_filter = @@ -237,11 +237,11 @@ void Java_org_rocksdb_Options_setCompactionFilterHandle( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompactionFilterFactoryHandle * Signature: (JJ)V */ -void JNICALL Java_org_rocksdb_Options_setCompactionFilterFactoryHandle( +void JNICALL Java_org_forstdb_Options_setCompactionFilterFactoryHandle( JNIEnv*, jobject, jlong jopt_handle, jlong jcompactionfilterfactory_handle) { auto* cff_factory = reinterpret_cast< @@ -252,11 +252,11 @@ void JNICALL Java_org_rocksdb_Options_setCompactionFilterFactoryHandle( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWriteBufferSize * Signature: (JJ)I */ -void Java_org_rocksdb_Options_setWriteBufferSize(JNIEnv* env, jobject, +void Java_org_forstdb_Options_setWriteBufferSize(JNIEnv* env, jobject, jlong jhandle, jlong jwrite_buffer_size) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( @@ -270,11 +270,11 @@ void Java_org_rocksdb_Options_setWriteBufferSize(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWriteBufferManager * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setWriteBufferManager( +void Java_org_forstdb_Options_setWriteBufferManager( JNIEnv*, jobject, jlong joptions_handle, jlong jwrite_buffer_manager_handle) { auto* write_buffer_manager = @@ -285,33 +285,33 @@ void Java_org_rocksdb_Options_setWriteBufferManager( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: writeBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_writeBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_writeBufferSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->write_buffer_size; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxWriteBufferNumber * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxWriteBufferNumber( +void Java_org_forstdb_Options_setMaxWriteBufferNumber( JNIEnv*, jobject, jlong jhandle, jint jmax_write_buffer_number) { reinterpret_cast(jhandle) ->max_write_buffer_number = jmax_write_buffer_number; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setStatistics * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setStatistics(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setStatistics(JNIEnv*, jobject, jlong jhandle, jlong jstatistics_handle) { auto* opt = reinterpret_cast(jhandle); auto* pSptr = @@ -321,11 +321,11 @@ void Java_org_rocksdb_Options_setStatistics(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: statistics * Signature: (J)J */ -jlong Java_org_rocksdb_Options_statistics(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_statistics(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); std::shared_ptr sptr = opt->statistics; if (sptr == nullptr) { @@ -338,77 +338,77 @@ jlong Java_org_rocksdb_Options_statistics(JNIEnv*, jobject, jlong jhandle) { } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxWriteBufferNumber * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxWriteBufferNumber(JNIEnv*, jobject, +jint Java_org_forstdb_Options_maxWriteBufferNumber(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_write_buffer_number; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: errorIfExists * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_errorIfExists(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_errorIfExists(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->error_if_exists; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setErrorIfExists * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setErrorIfExists(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setErrorIfExists(JNIEnv*, jobject, jlong jhandle, jboolean error_if_exists) { reinterpret_cast(jhandle)->error_if_exists = static_cast(error_if_exists); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: paranoidChecks * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_paranoidChecks(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_paranoidChecks(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->paranoid_checks; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setParanoidChecks * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setParanoidChecks(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setParanoidChecks(JNIEnv*, jobject, jlong jhandle, jboolean paranoid_checks) { reinterpret_cast(jhandle)->paranoid_checks = static_cast(paranoid_checks); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setEnv * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setEnv(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setEnv(JNIEnv*, jobject, jlong jhandle, jlong jenv) { reinterpret_cast(jhandle)->env = reinterpret_cast(jenv); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxTotalWalSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMaxTotalWalSize(JNIEnv*, jobject, +void Java_org_forstdb_Options_setMaxTotalWalSize(JNIEnv*, jobject, jlong jhandle, jlong jmax_total_wal_size) { reinterpret_cast(jhandle)->max_total_wal_size = @@ -416,84 +416,84 @@ void Java_org_rocksdb_Options_setMaxTotalWalSize(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxTotalWalSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_maxTotalWalSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_maxTotalWalSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_total_wal_size; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxOpenFiles * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxOpenFiles(JNIEnv*, jobject, jlong jhandle) { +jint Java_org_forstdb_Options_maxOpenFiles(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->max_open_files; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxOpenFiles * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxOpenFiles(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setMaxOpenFiles(JNIEnv*, jobject, jlong jhandle, jint max_open_files) { reinterpret_cast(jhandle)->max_open_files = static_cast(max_open_files); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxFileOpeningThreads * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxFileOpeningThreads( +void Java_org_forstdb_Options_setMaxFileOpeningThreads( JNIEnv*, jobject, jlong jhandle, jint jmax_file_opening_threads) { reinterpret_cast(jhandle) ->max_file_opening_threads = static_cast(jmax_file_opening_threads); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxFileOpeningThreads * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxFileOpeningThreads(JNIEnv*, jobject, +jint Java_org_forstdb_Options_maxFileOpeningThreads(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_file_opening_threads); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: useFsync * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_useFsync(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_Options_useFsync(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->use_fsync; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setUseFsync * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setUseFsync(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setUseFsync(JNIEnv*, jobject, jlong jhandle, jboolean use_fsync) { reinterpret_cast(jhandle)->use_fsync = static_cast(use_fsync); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setDbPaths * Signature: (J[Ljava/lang/String;[J)V */ -void Java_org_rocksdb_Options_setDbPaths(JNIEnv* env, jobject, jlong jhandle, +void Java_org_forstdb_Options_setDbPaths(JNIEnv* env, jobject, jlong jhandle, jobjectArray jpaths, jlongArray jtarget_sizes) { std::vector db_paths; @@ -535,21 +535,21 @@ void Java_org_rocksdb_Options_setDbPaths(JNIEnv* env, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: dbPathsLen * Signature: (J)J */ -jlong Java_org_rocksdb_Options_dbPathsLen(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_dbPathsLen(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->db_paths.size()); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: dbPaths * Signature: (J[Ljava/lang/String;[J)V */ -void Java_org_rocksdb_Options_dbPaths(JNIEnv* env, jobject, jlong jhandle, +void Java_org_forstdb_Options_dbPaths(JNIEnv* env, jobject, jlong jhandle, jobjectArray jpaths, jlongArray jtarget_sizes) { jboolean is_copy; @@ -586,22 +586,22 @@ void Java_org_rocksdb_Options_dbPaths(JNIEnv* env, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: dbLogDir * Signature: (J)Ljava/lang/String */ -jstring Java_org_rocksdb_Options_dbLogDir(JNIEnv* env, jobject, jlong jhandle) { +jstring Java_org_forstdb_Options_dbLogDir(JNIEnv* env, jobject, jlong jhandle) { return env->NewStringUTF( reinterpret_cast(jhandle) ->db_log_dir.c_str()); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setDbLogDir * Signature: (JLjava/lang/String)V */ -void Java_org_rocksdb_Options_setDbLogDir(JNIEnv* env, jobject, jlong jhandle, +void Java_org_forstdb_Options_setDbLogDir(JNIEnv* env, jobject, jlong jhandle, jstring jdb_log_dir) { const char* log_dir = env->GetStringUTFChars(jdb_log_dir, nullptr); if (log_dir == nullptr) { @@ -614,21 +614,21 @@ void Java_org_rocksdb_Options_setDbLogDir(JNIEnv* env, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: walDir * Signature: (J)Ljava/lang/String */ -jstring Java_org_rocksdb_Options_walDir(JNIEnv* env, jobject, jlong jhandle) { +jstring Java_org_forstdb_Options_walDir(JNIEnv* env, jobject, jlong jhandle) { return env->NewStringUTF( reinterpret_cast(jhandle)->wal_dir.c_str()); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWalDir * Signature: (JLjava/lang/String)V */ -void Java_org_rocksdb_Options_setWalDir(JNIEnv* env, jobject, jlong jhandle, +void Java_org_forstdb_Options_setWalDir(JNIEnv* env, jobject, jlong jhandle, jstring jwal_dir) { const char* wal_dir = env->GetStringUTFChars(jwal_dir, nullptr); if (wal_dir == nullptr) { @@ -641,22 +641,22 @@ void Java_org_rocksdb_Options_setWalDir(JNIEnv* env, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: deleteObsoleteFilesPeriodMicros * Signature: (J)J */ -jlong Java_org_rocksdb_Options_deleteObsoleteFilesPeriodMicros(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_deleteObsoleteFilesPeriodMicros(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->delete_obsolete_files_period_micros; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setDeleteObsoleteFilesPeriodMicros * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setDeleteObsoleteFilesPeriodMicros(JNIEnv*, +void Java_org_forstdb_Options_setDeleteObsoleteFilesPeriodMicros(JNIEnv*, jobject, jlong jhandle, jlong micros) { @@ -665,22 +665,22 @@ void Java_org_rocksdb_Options_setDeleteObsoleteFilesPeriodMicros(JNIEnv*, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxBackgroundCompactions * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxBackgroundCompactions(JNIEnv*, jobject, +jint Java_org_forstdb_Options_maxBackgroundCompactions(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_background_compactions; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxBackgroundCompactions * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxBackgroundCompactions(JNIEnv*, jobject, +void Java_org_forstdb_Options_setMaxBackgroundCompactions(JNIEnv*, jobject, jlong jhandle, jint max) { reinterpret_cast(jhandle) @@ -688,66 +688,66 @@ void Java_org_rocksdb_Options_setMaxBackgroundCompactions(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxSubcompactions * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxSubcompactions(JNIEnv*, jobject, +void Java_org_forstdb_Options_setMaxSubcompactions(JNIEnv*, jobject, jlong jhandle, jint max) { reinterpret_cast(jhandle)->max_subcompactions = static_cast(max); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxSubcompactions * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxSubcompactions(JNIEnv*, jobject, +jint Java_org_forstdb_Options_maxSubcompactions(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_subcompactions; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxBackgroundFlushes * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxBackgroundFlushes(JNIEnv*, jobject, +jint Java_org_forstdb_Options_maxBackgroundFlushes(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_background_flushes; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxBackgroundFlushes * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxBackgroundFlushes( +void Java_org_forstdb_Options_setMaxBackgroundFlushes( JNIEnv*, jobject, jlong jhandle, jint max_background_flushes) { reinterpret_cast(jhandle) ->max_background_flushes = static_cast(max_background_flushes); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxBackgroundJobs * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxBackgroundJobs(JNIEnv*, jobject, +jint Java_org_forstdb_Options_maxBackgroundJobs(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_background_jobs; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxBackgroundJobs * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxBackgroundJobs(JNIEnv*, jobject, +void Java_org_forstdb_Options_setMaxBackgroundJobs(JNIEnv*, jobject, jlong jhandle, jint max_background_jobs) { reinterpret_cast(jhandle)->max_background_jobs = @@ -755,21 +755,21 @@ void Java_org_rocksdb_Options_setMaxBackgroundJobs(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxLogFileSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_maxLogFileSize(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_maxLogFileSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_log_file_size; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxLogFileSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMaxLogFileSize(JNIEnv* env, jobject, +void Java_org_forstdb_Options_setMaxLogFileSize(JNIEnv* env, jobject, jlong jhandle, jlong max_log_file_size) { auto s = @@ -783,22 +783,22 @@ void Java_org_rocksdb_Options_setMaxLogFileSize(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: logFileTimeToRoll * Signature: (J)J */ -jlong Java_org_rocksdb_Options_logFileTimeToRoll(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_logFileTimeToRoll(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->log_file_time_to_roll; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLogFileTimeToRoll * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setLogFileTimeToRoll( +void Java_org_forstdb_Options_setLogFileTimeToRoll( JNIEnv* env, jobject, jlong jhandle, jlong log_file_time_to_roll) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( log_file_time_to_roll); @@ -811,21 +811,21 @@ void Java_org_rocksdb_Options_setLogFileTimeToRoll( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: keepLogFileNum * Signature: (J)J */ -jlong Java_org_rocksdb_Options_keepLogFileNum(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_keepLogFileNum(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->keep_log_file_num; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setKeepLogFileNum * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setKeepLogFileNum(JNIEnv* env, jobject, +void Java_org_forstdb_Options_setKeepLogFileNum(JNIEnv* env, jobject, jlong jhandle, jlong keep_log_file_num) { auto s = @@ -839,22 +839,22 @@ void Java_org_rocksdb_Options_setKeepLogFileNum(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: recycleLogFileNum * Signature: (J)J */ -jlong Java_org_rocksdb_Options_recycleLogFileNum(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_recycleLogFileNum(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->recycle_log_file_num; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setRecycleLogFileNum * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setRecycleLogFileNum(JNIEnv* env, jobject, +void Java_org_forstdb_Options_setRecycleLogFileNum(JNIEnv* env, jobject, jlong jhandle, jlong recycle_log_file_num) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( @@ -868,11 +868,11 @@ void Java_org_rocksdb_Options_setRecycleLogFileNum(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxManifestFileSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_maxManifestFileSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_maxManifestFileSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_manifest_file_size; @@ -882,7 +882,7 @@ jlong Java_org_rocksdb_Options_maxManifestFileSize(JNIEnv*, jobject, * Method: memTableFactoryName * Signature: (J)Ljava/lang/String */ -jstring Java_org_rocksdb_Options_memTableFactoryName(JNIEnv* env, jobject, +jstring Java_org_forstdb_Options_memTableFactoryName(JNIEnv* env, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); ROCKSDB_NAMESPACE::MemTableRepFactory* tf = opt->memtable_factory.get(); @@ -950,11 +950,11 @@ rocksdb_convert_cf_paths_from_java_helper(JNIEnv* env, jobjectArray path_array, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCfPaths * Signature: (J[Ljava/lang/String;[J)V */ -void Java_org_rocksdb_Options_setCfPaths(JNIEnv* env, jclass, jlong jhandle, +void Java_org_forstdb_Options_setCfPaths(JNIEnv* env, jclass, jlong jhandle, jobjectArray path_array, jlongArray size_array) { auto* options = reinterpret_cast(jhandle); @@ -968,11 +968,11 @@ void Java_org_rocksdb_Options_setCfPaths(JNIEnv* env, jclass, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: cfPathsLen * Signature: (J)J */ -jlong Java_org_rocksdb_Options_cfPathsLen(JNIEnv*, jclass, jlong jhandle) { +jlong Java_org_forstdb_Options_cfPathsLen(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->cf_paths.size()); } @@ -1017,11 +1017,11 @@ static void rocksdb_convert_cf_paths_to_java_helper(JNIEnv* env, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: cfPaths * Signature: (J[Ljava/lang/String;[J)V */ -void Java_org_rocksdb_Options_cfPaths(JNIEnv* env, jclass, jlong jhandle, +void Java_org_forstdb_Options_cfPaths(JNIEnv* env, jclass, jlong jhandle, jobjectArray jpaths, jlongArray jtarget_sizes) { rocksdb_convert_cf_paths_to_java_helper( @@ -1029,11 +1029,11 @@ void Java_org_rocksdb_Options_cfPaths(JNIEnv* env, jclass, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxManifestFileSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMaxManifestFileSize( +void Java_org_forstdb_Options_setMaxManifestFileSize( JNIEnv*, jobject, jlong jhandle, jlong max_manifest_file_size) { reinterpret_cast(jhandle) ->max_manifest_file_size = static_cast(max_manifest_file_size); @@ -1043,7 +1043,7 @@ void Java_org_rocksdb_Options_setMaxManifestFileSize( * Method: setMemTableFactory * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMemTableFactory(JNIEnv*, jobject, +void Java_org_forstdb_Options_setMemTableFactory(JNIEnv*, jobject, jlong jhandle, jlong jfactory_handle) { reinterpret_cast(jhandle) @@ -1053,11 +1053,11 @@ void Java_org_rocksdb_Options_setMemTableFactory(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setRateLimiter * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setRateLimiter(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setRateLimiter(JNIEnv*, jobject, jlong jhandle, jlong jrate_limiter_handle) { std::shared_ptr* pRateLimiter = reinterpret_cast*>( @@ -1067,11 +1067,11 @@ void Java_org_rocksdb_Options_setRateLimiter(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setSstFileManager * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setSstFileManager( +void Java_org_forstdb_Options_setSstFileManager( JNIEnv*, jobject, jlong jhandle, jlong jsst_file_manager_handle) { auto* sptr_sst_file_manager = reinterpret_cast*>( @@ -1081,11 +1081,11 @@ void Java_org_rocksdb_Options_setSstFileManager( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLogger * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setLogger(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setLogger(JNIEnv*, jobject, jlong jhandle, jlong jlogger_handle) { std::shared_ptr* pLogger = reinterpret_cast*>( @@ -1094,43 +1094,43 @@ void Java_org_rocksdb_Options_setLogger(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setInfoLogLevel * Signature: (JB)V */ -void Java_org_rocksdb_Options_setInfoLogLevel(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setInfoLogLevel(JNIEnv*, jobject, jlong jhandle, jbyte jlog_level) { reinterpret_cast(jhandle)->info_log_level = static_cast(jlog_level); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: infoLogLevel * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_infoLogLevel(JNIEnv*, jobject, jlong jhandle) { +jbyte Java_org_forstdb_Options_infoLogLevel(JNIEnv*, jobject, jlong jhandle) { return static_cast( reinterpret_cast(jhandle)->info_log_level); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: tableCacheNumshardbits * Signature: (J)I */ -jint Java_org_rocksdb_Options_tableCacheNumshardbits(JNIEnv*, jobject, +jint Java_org_forstdb_Options_tableCacheNumshardbits(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->table_cache_numshardbits; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setTableCacheNumshardbits * Signature: (JI)V */ -void Java_org_rocksdb_Options_setTableCacheNumshardbits( +void Java_org_forstdb_Options_setTableCacheNumshardbits( JNIEnv*, jobject, jlong jhandle, jint table_cache_numshardbits) { reinterpret_cast(jhandle) ->table_cache_numshardbits = static_cast(table_cache_numshardbits); @@ -1140,7 +1140,7 @@ void Java_org_rocksdb_Options_setTableCacheNumshardbits( * Method: useFixedLengthPrefixExtractor * Signature: (JI)V */ -void Java_org_rocksdb_Options_useFixedLengthPrefixExtractor( +void Java_org_forstdb_Options_useFixedLengthPrefixExtractor( JNIEnv*, jobject, jlong jhandle, jint jprefix_length) { reinterpret_cast(jhandle) ->prefix_extractor.reset(ROCKSDB_NAMESPACE::NewFixedPrefixTransform( @@ -1151,7 +1151,7 @@ void Java_org_rocksdb_Options_useFixedLengthPrefixExtractor( * Method: useCappedPrefixExtractor * Signature: (JI)V */ -void Java_org_rocksdb_Options_useCappedPrefixExtractor(JNIEnv*, jobject, +void Java_org_forstdb_Options_useCappedPrefixExtractor(JNIEnv*, jobject, jlong jhandle, jint jprefix_length) { reinterpret_cast(jhandle) @@ -1160,53 +1160,53 @@ void Java_org_rocksdb_Options_useCappedPrefixExtractor(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: walTtlSeconds * Signature: (J)J */ -jlong Java_org_rocksdb_Options_walTtlSeconds(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_walTtlSeconds(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->WAL_ttl_seconds; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWalTtlSeconds * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setWalTtlSeconds(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setWalTtlSeconds(JNIEnv*, jobject, jlong jhandle, jlong WAL_ttl_seconds) { reinterpret_cast(jhandle)->WAL_ttl_seconds = static_cast(WAL_ttl_seconds); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: walTtlSeconds * Signature: (J)J */ -jlong Java_org_rocksdb_Options_walSizeLimitMB(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_walSizeLimitMB(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->WAL_size_limit_MB; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWalSizeLimitMB * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setWalSizeLimitMB(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setWalSizeLimitMB(JNIEnv*, jobject, jlong jhandle, jlong WAL_size_limit_MB) { reinterpret_cast(jhandle)->WAL_size_limit_MB = static_cast(WAL_size_limit_MB); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxWriteBatchGroupSizeBytes * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMaxWriteBatchGroupSizeBytes( +void Java_org_forstdb_Options_setMaxWriteBatchGroupSizeBytes( JNIEnv*, jclass, jlong jhandle, jlong jmax_write_batch_group_size_bytes) { auto* opt = reinterpret_cast(jhandle); opt->max_write_batch_group_size_bytes = @@ -1214,33 +1214,33 @@ void Java_org_rocksdb_Options_setMaxWriteBatchGroupSizeBytes( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxWriteBatchGroupSizeBytes * Signature: (J)J */ -jlong Java_org_rocksdb_Options_maxWriteBatchGroupSizeBytes(JNIEnv*, jclass, +jlong Java_org_forstdb_Options_maxWriteBatchGroupSizeBytes(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_write_batch_group_size_bytes); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: manifestPreallocationSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_manifestPreallocationSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_manifestPreallocationSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->manifest_preallocation_size; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setManifestPreallocationSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setManifestPreallocationSize( +void Java_org_forstdb_Options_setManifestPreallocationSize( JNIEnv* env, jobject, jlong jhandle, jlong preallocation_size) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( preallocation_size); @@ -1256,7 +1256,7 @@ void Java_org_rocksdb_Options_setManifestPreallocationSize( * Method: setTableFactory * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setTableFactory(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setTableFactory(JNIEnv*, jobject, jlong jhandle, jlong jtable_factory_handle) { auto* options = reinterpret_cast(jhandle); auto* table_factory = @@ -1268,7 +1268,7 @@ void Java_org_rocksdb_Options_setTableFactory(JNIEnv*, jobject, jlong jhandle, * Method: setSstPartitionerFactory * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setSstPartitionerFactory(JNIEnv*, jobject, +void Java_org_forstdb_Options_setSstPartitionerFactory(JNIEnv*, jobject, jlong jhandle, jlong factory_handle) { auto* options = reinterpret_cast(jhandle); @@ -1279,11 +1279,11 @@ void Java_org_rocksdb_Options_setSstPartitionerFactory(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompactionThreadLimiter * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setCompactionThreadLimiter( +void Java_org_forstdb_Options_setCompactionThreadLimiter( JNIEnv*, jclass, jlong jhandle, jlong jlimiter_handle) { auto* options = reinterpret_cast(jhandle); auto* limiter = reinterpret_cast< @@ -1293,44 +1293,44 @@ void Java_org_rocksdb_Options_setCompactionThreadLimiter( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: allowMmapReads * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_allowMmapReads(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_allowMmapReads(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->allow_mmap_reads; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAllowMmapReads * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAllowMmapReads(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setAllowMmapReads(JNIEnv*, jobject, jlong jhandle, jboolean allow_mmap_reads) { reinterpret_cast(jhandle)->allow_mmap_reads = static_cast(allow_mmap_reads); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: allowMmapWrites * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_allowMmapWrites(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_allowMmapWrites(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->allow_mmap_writes; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAllowMmapWrites * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAllowMmapWrites(JNIEnv*, jobject, +void Java_org_forstdb_Options_setAllowMmapWrites(JNIEnv*, jobject, jlong jhandle, jboolean allow_mmap_writes) { reinterpret_cast(jhandle)->allow_mmap_writes = @@ -1338,44 +1338,44 @@ void Java_org_rocksdb_Options_setAllowMmapWrites(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: useDirectReads * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_useDirectReads(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_useDirectReads(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->use_direct_reads; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setUseDirectReads * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setUseDirectReads(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setUseDirectReads(JNIEnv*, jobject, jlong jhandle, jboolean use_direct_reads) { reinterpret_cast(jhandle)->use_direct_reads = static_cast(use_direct_reads); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: useDirectIoForFlushAndCompaction * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_useDirectIoForFlushAndCompaction( +jboolean Java_org_forstdb_Options_useDirectIoForFlushAndCompaction( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->use_direct_io_for_flush_and_compaction; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setUseDirectIoForFlushAndCompaction * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setUseDirectIoForFlushAndCompaction( +void Java_org_forstdb_Options_setUseDirectIoForFlushAndCompaction( JNIEnv*, jobject, jlong jhandle, jboolean use_direct_io_for_flush_and_compaction) { reinterpret_cast(jhandle) @@ -1384,44 +1384,44 @@ void Java_org_rocksdb_Options_setUseDirectIoForFlushAndCompaction( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAllowFAllocate * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAllowFAllocate(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setAllowFAllocate(JNIEnv*, jobject, jlong jhandle, jboolean jallow_fallocate) { reinterpret_cast(jhandle)->allow_fallocate = static_cast(jallow_fallocate); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: allowFAllocate * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_allowFAllocate(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_allowFAllocate(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->allow_fallocate); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: isFdCloseOnExec * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_isFdCloseOnExec(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_isFdCloseOnExec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->is_fd_close_on_exec; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setIsFdCloseOnExec * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setIsFdCloseOnExec(JNIEnv*, jobject, +void Java_org_forstdb_Options_setIsFdCloseOnExec(JNIEnv*, jobject, jlong jhandle, jboolean is_fd_close_on_exec) { reinterpret_cast(jhandle)->is_fd_close_on_exec = @@ -1429,22 +1429,22 @@ void Java_org_rocksdb_Options_setIsFdCloseOnExec(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: statsDumpPeriodSec * Signature: (J)I */ -jint Java_org_rocksdb_Options_statsDumpPeriodSec(JNIEnv*, jobject, +jint Java_org_forstdb_Options_statsDumpPeriodSec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->stats_dump_period_sec; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setStatsDumpPeriodSec * Signature: (JI)V */ -void Java_org_rocksdb_Options_setStatsDumpPeriodSec( +void Java_org_forstdb_Options_setStatsDumpPeriodSec( JNIEnv*, jobject, jlong jhandle, jint jstats_dump_period_sec) { reinterpret_cast(jhandle) ->stats_dump_period_sec = @@ -1452,22 +1452,22 @@ void Java_org_rocksdb_Options_setStatsDumpPeriodSec( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: statsPersistPeriodSec * Signature: (J)I */ -jint Java_org_rocksdb_Options_statsPersistPeriodSec(JNIEnv*, jobject, +jint Java_org_forstdb_Options_statsPersistPeriodSec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->stats_persist_period_sec; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setStatsPersistPeriodSec * Signature: (JI)V */ -void Java_org_rocksdb_Options_setStatsPersistPeriodSec( +void Java_org_forstdb_Options_setStatsPersistPeriodSec( JNIEnv*, jobject, jlong jhandle, jint jstats_persist_period_sec) { reinterpret_cast(jhandle) ->stats_persist_period_sec = @@ -1475,22 +1475,22 @@ void Java_org_rocksdb_Options_setStatsPersistPeriodSec( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: statsHistoryBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_statsHistoryBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_statsHistoryBufferSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->stats_history_buffer_size; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setStatsHistoryBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setStatsHistoryBufferSize( +void Java_org_forstdb_Options_setStatsHistoryBufferSize( JNIEnv*, jobject, jlong jhandle, jlong jstats_history_buffer_size) { reinterpret_cast(jhandle) ->stats_history_buffer_size = @@ -1498,55 +1498,55 @@ void Java_org_rocksdb_Options_setStatsHistoryBufferSize( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: adviseRandomOnOpen * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_adviseRandomOnOpen(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_adviseRandomOnOpen(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->advise_random_on_open; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAdviseRandomOnOpen * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAdviseRandomOnOpen( +void Java_org_forstdb_Options_setAdviseRandomOnOpen( JNIEnv*, jobject, jlong jhandle, jboolean advise_random_on_open) { reinterpret_cast(jhandle) ->advise_random_on_open = static_cast(advise_random_on_open); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setDbWriteBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setDbWriteBufferSize( +void Java_org_forstdb_Options_setDbWriteBufferSize( JNIEnv*, jobject, jlong jhandle, jlong jdb_write_buffer_size) { auto* opt = reinterpret_cast(jhandle); opt->db_write_buffer_size = static_cast(jdb_write_buffer_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: dbWriteBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_dbWriteBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_dbWriteBufferSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->db_write_buffer_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAccessHintOnCompactionStart * Signature: (JB)V */ -void Java_org_rocksdb_Options_setAccessHintOnCompactionStart( +void Java_org_forstdb_Options_setAccessHintOnCompactionStart( JNIEnv*, jobject, jlong jhandle, jbyte jaccess_hint_value) { auto* opt = reinterpret_cast(jhandle); opt->access_hint_on_compaction_start = @@ -1554,11 +1554,11 @@ void Java_org_rocksdb_Options_setAccessHintOnCompactionStart( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: accessHintOnCompactionStart * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_accessHintOnCompactionStart(JNIEnv*, jobject, +jbyte Java_org_forstdb_Options_accessHintOnCompactionStart(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::AccessHintJni::toJavaAccessHint( @@ -1566,11 +1566,11 @@ jbyte Java_org_rocksdb_Options_accessHintOnCompactionStart(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompactionReadaheadSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setCompactionReadaheadSize( +void Java_org_forstdb_Options_setCompactionReadaheadSize( JNIEnv*, jobject, jlong jhandle, jlong jcompaction_readahead_size) { auto* opt = reinterpret_cast(jhandle); opt->compaction_readahead_size = @@ -1578,22 +1578,22 @@ void Java_org_rocksdb_Options_setCompactionReadaheadSize( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: compactionReadaheadSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_compactionReadaheadSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_compactionReadaheadSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->compaction_readahead_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setRandomAccessMaxBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setRandomAccessMaxBufferSize( +void Java_org_forstdb_Options_setRandomAccessMaxBufferSize( JNIEnv*, jobject, jlong jhandle, jlong jrandom_access_max_buffer_size) { auto* opt = reinterpret_cast(jhandle); opt->random_access_max_buffer_size = @@ -1601,22 +1601,22 @@ void Java_org_rocksdb_Options_setRandomAccessMaxBufferSize( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: randomAccessMaxBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_randomAccessMaxBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_randomAccessMaxBufferSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->random_access_max_buffer_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWritableFileMaxBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setWritableFileMaxBufferSize( +void Java_org_forstdb_Options_setWritableFileMaxBufferSize( JNIEnv*, jobject, jlong jhandle, jlong jwritable_file_max_buffer_size) { auto* opt = reinterpret_cast(jhandle); opt->writable_file_max_buffer_size = @@ -1624,33 +1624,33 @@ void Java_org_rocksdb_Options_setWritableFileMaxBufferSize( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: writableFileMaxBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_writableFileMaxBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_writableFileMaxBufferSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->writable_file_max_buffer_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: useAdaptiveMutex * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_useAdaptiveMutex(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_useAdaptiveMutex(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->use_adaptive_mutex; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setUseAdaptiveMutex * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setUseAdaptiveMutex(JNIEnv*, jobject, +void Java_org_forstdb_Options_setUseAdaptiveMutex(JNIEnv*, jobject, jlong jhandle, jboolean use_adaptive_mutex) { reinterpret_cast(jhandle)->use_adaptive_mutex = @@ -1658,31 +1658,31 @@ void Java_org_rocksdb_Options_setUseAdaptiveMutex(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: bytesPerSync * Signature: (J)J */ -jlong Java_org_rocksdb_Options_bytesPerSync(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_bytesPerSync(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->bytes_per_sync; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBytesPerSync * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setBytesPerSync(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setBytesPerSync(JNIEnv*, jobject, jlong jhandle, jlong bytes_per_sync) { reinterpret_cast(jhandle)->bytes_per_sync = static_cast(bytes_per_sync); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWalBytesPerSync * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setWalBytesPerSync(JNIEnv*, jobject, +void Java_org_forstdb_Options_setWalBytesPerSync(JNIEnv*, jobject, jlong jhandle, jlong jwal_bytes_per_sync) { reinterpret_cast(jhandle)->wal_bytes_per_sync = @@ -1690,33 +1690,33 @@ void Java_org_rocksdb_Options_setWalBytesPerSync(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: walBytesPerSync * Signature: (J)J */ -jlong Java_org_rocksdb_Options_walBytesPerSync(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_walBytesPerSync(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->wal_bytes_per_sync); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setStrictBytesPerSync * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setStrictBytesPerSync( +void Java_org_forstdb_Options_setStrictBytesPerSync( JNIEnv*, jobject, jlong jhandle, jboolean jstrict_bytes_per_sync) { reinterpret_cast(jhandle) ->strict_bytes_per_sync = jstrict_bytes_per_sync == JNI_TRUE; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: strictBytesPerSync * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_strictBytesPerSync(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_strictBytesPerSync(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->strict_bytes_per_sync); @@ -1746,11 +1746,11 @@ static void rocksdb_set_event_listeners_helper( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setEventListeners * Signature: (J[J)V */ -void Java_org_rocksdb_Options_setEventListeners(JNIEnv* env, jclass, +void Java_org_forstdb_Options_setEventListeners(JNIEnv* env, jclass, jlong jhandle, jlongArray jlistener_array) { auto* opt = reinterpret_cast(jhandle); @@ -1782,44 +1782,44 @@ static jobjectArray rocksdb_get_event_listeners_helper( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: eventListeners - * Signature: (J)[Lorg/rocksdb/AbstractEventListener; + * Signature: (J)[Lorg/forstdb/AbstractEventListener; */ -jobjectArray Java_org_rocksdb_Options_eventListeners(JNIEnv* env, jclass, +jobjectArray Java_org_forstdb_Options_eventListeners(JNIEnv* env, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return rocksdb_get_event_listeners_helper(env, opt->listeners); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setEnableThreadTracking * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setEnableThreadTracking( +void Java_org_forstdb_Options_setEnableThreadTracking( JNIEnv*, jobject, jlong jhandle, jboolean jenable_thread_tracking) { auto* opt = reinterpret_cast(jhandle); opt->enable_thread_tracking = static_cast(jenable_thread_tracking); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: enableThreadTracking * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_enableThreadTracking(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_enableThreadTracking(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->enable_thread_tracking); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setDelayedWriteRate * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setDelayedWriteRate(JNIEnv*, jobject, +void Java_org_forstdb_Options_setDelayedWriteRate(JNIEnv*, jobject, jlong jhandle, jlong jdelayed_write_rate) { auto* opt = reinterpret_cast(jhandle); @@ -1827,66 +1827,66 @@ void Java_org_rocksdb_Options_setDelayedWriteRate(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: delayedWriteRate * Signature: (J)J */ -jlong Java_org_rocksdb_Options_delayedWriteRate(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_delayedWriteRate(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->delayed_write_rate); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setEnablePipelinedWrite * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setEnablePipelinedWrite( +void Java_org_forstdb_Options_setEnablePipelinedWrite( JNIEnv*, jobject, jlong jhandle, jboolean jenable_pipelined_write) { auto* opt = reinterpret_cast(jhandle); opt->enable_pipelined_write = jenable_pipelined_write == JNI_TRUE; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: enablePipelinedWrite * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_enablePipelinedWrite(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_enablePipelinedWrite(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->enable_pipelined_write); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setUnorderedWrite * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setUnorderedWrite(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setUnorderedWrite(JNIEnv*, jobject, jlong jhandle, jboolean unordered_write) { reinterpret_cast(jhandle)->unordered_write = static_cast(unordered_write); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: unorderedWrite * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_unorderedWrite(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_unorderedWrite(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->unordered_write; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAllowConcurrentMemtableWrite * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAllowConcurrentMemtableWrite(JNIEnv*, jobject, +void Java_org_forstdb_Options_setAllowConcurrentMemtableWrite(JNIEnv*, jobject, jlong jhandle, jboolean allow) { reinterpret_cast(jhandle) @@ -1894,44 +1894,44 @@ void Java_org_rocksdb_Options_setAllowConcurrentMemtableWrite(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: allowConcurrentMemtableWrite * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_allowConcurrentMemtableWrite(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_allowConcurrentMemtableWrite(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->allow_concurrent_memtable_write; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setEnableWriteThreadAdaptiveYield * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setEnableWriteThreadAdaptiveYield( +void Java_org_forstdb_Options_setEnableWriteThreadAdaptiveYield( JNIEnv*, jobject, jlong jhandle, jboolean yield) { reinterpret_cast(jhandle) ->enable_write_thread_adaptive_yield = static_cast(yield); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: enableWriteThreadAdaptiveYield * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_enableWriteThreadAdaptiveYield( +jboolean Java_org_forstdb_Options_enableWriteThreadAdaptiveYield( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->enable_write_thread_adaptive_yield; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWriteThreadMaxYieldUsec * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setWriteThreadMaxYieldUsec(JNIEnv*, jobject, +void Java_org_forstdb_Options_setWriteThreadMaxYieldUsec(JNIEnv*, jobject, jlong jhandle, jlong max) { reinterpret_cast(jhandle) @@ -1939,22 +1939,22 @@ void Java_org_rocksdb_Options_setWriteThreadMaxYieldUsec(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: writeThreadMaxYieldUsec * Signature: (J)J */ -jlong Java_org_rocksdb_Options_writeThreadMaxYieldUsec(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_writeThreadMaxYieldUsec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->write_thread_max_yield_usec; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWriteThreadSlowYieldUsec * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setWriteThreadSlowYieldUsec(JNIEnv*, jobject, +void Java_org_forstdb_Options_setWriteThreadSlowYieldUsec(JNIEnv*, jobject, jlong jhandle, jlong slow) { reinterpret_cast(jhandle) @@ -1962,22 +1962,22 @@ void Java_org_rocksdb_Options_setWriteThreadSlowYieldUsec(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: writeThreadSlowYieldUsec * Signature: (J)J */ -jlong Java_org_rocksdb_Options_writeThreadSlowYieldUsec(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_writeThreadSlowYieldUsec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->write_thread_slow_yield_usec; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setSkipStatsUpdateOnDbOpen * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setSkipStatsUpdateOnDbOpen( +void Java_org_forstdb_Options_setSkipStatsUpdateOnDbOpen( JNIEnv*, jobject, jlong jhandle, jboolean jskip_stats_update_on_db_open) { auto* opt = reinterpret_cast(jhandle); opt->skip_stats_update_on_db_open = @@ -1985,22 +1985,22 @@ void Java_org_rocksdb_Options_setSkipStatsUpdateOnDbOpen( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: skipStatsUpdateOnDbOpen * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_skipStatsUpdateOnDbOpen(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_skipStatsUpdateOnDbOpen(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->skip_stats_update_on_db_open); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setSkipCheckingSstFileSizesOnDbOpen * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setSkipCheckingSstFileSizesOnDbOpen( +void Java_org_forstdb_Options_setSkipCheckingSstFileSizesOnDbOpen( JNIEnv*, jclass, jlong jhandle, jboolean jskip_checking_sst_file_sizes_on_db_open) { auto* opt = reinterpret_cast(jhandle); @@ -2009,22 +2009,22 @@ void Java_org_rocksdb_Options_setSkipCheckingSstFileSizesOnDbOpen( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: skipCheckingSstFileSizesOnDbOpen * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_skipCheckingSstFileSizesOnDbOpen( +jboolean Java_org_forstdb_Options_skipCheckingSstFileSizesOnDbOpen( JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->skip_checking_sst_file_sizes_on_db_open); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWalRecoveryMode * Signature: (JB)V */ -void Java_org_rocksdb_Options_setWalRecoveryMode( +void Java_org_forstdb_Options_setWalRecoveryMode( JNIEnv*, jobject, jlong jhandle, jbyte jwal_recovery_mode_value) { auto* opt = reinterpret_cast(jhandle); opt->wal_recovery_mode = @@ -2033,11 +2033,11 @@ void Java_org_rocksdb_Options_setWalRecoveryMode( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: walRecoveryMode * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_walRecoveryMode(JNIEnv*, jobject, +jbyte Java_org_forstdb_Options_walRecoveryMode(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::WALRecoveryModeJni::toJavaWALRecoveryMode( @@ -2045,32 +2045,32 @@ jbyte Java_org_rocksdb_Options_walRecoveryMode(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAllow2pc * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAllow2pc(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setAllow2pc(JNIEnv*, jobject, jlong jhandle, jboolean jallow_2pc) { auto* opt = reinterpret_cast(jhandle); opt->allow_2pc = static_cast(jallow_2pc); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: allow2pc * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_allow2pc(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_Options_allow2pc(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->allow_2pc); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setRowCache * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setRowCache(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setRowCache(JNIEnv*, jobject, jlong jhandle, jlong jrow_cache_handle) { auto* opt = reinterpret_cast(jhandle); auto* row_cache = @@ -2080,11 +2080,11 @@ void Java_org_rocksdb_Options_setRowCache(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWalFilter * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setWalFilter(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setWalFilter(JNIEnv*, jobject, jlong jhandle, jlong jwal_filter_handle) { auto* opt = reinterpret_cast(jhandle); auto* wal_filter = reinterpret_cast( @@ -2093,11 +2093,11 @@ void Java_org_rocksdb_Options_setWalFilter(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setFailIfOptionsFileError * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setFailIfOptionsFileError( +void Java_org_forstdb_Options_setFailIfOptionsFileError( JNIEnv*, jobject, jlong jhandle, jboolean jfail_if_options_file_error) { auto* opt = reinterpret_cast(jhandle); opt->fail_if_options_file_error = @@ -2105,22 +2105,22 @@ void Java_org_rocksdb_Options_setFailIfOptionsFileError( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: failIfOptionsFileError * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_failIfOptionsFileError(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_failIfOptionsFileError(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->fail_if_options_file_error); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setDumpMallocStats * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setDumpMallocStats(JNIEnv*, jobject, +void Java_org_forstdb_Options_setDumpMallocStats(JNIEnv*, jobject, jlong jhandle, jboolean jdump_malloc_stats) { auto* opt = reinterpret_cast(jhandle); @@ -2128,22 +2128,22 @@ void Java_org_rocksdb_Options_setDumpMallocStats(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: dumpMallocStats * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_dumpMallocStats(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_dumpMallocStats(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->dump_malloc_stats); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAvoidFlushDuringRecovery * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAvoidFlushDuringRecovery( +void Java_org_forstdb_Options_setAvoidFlushDuringRecovery( JNIEnv*, jobject, jlong jhandle, jboolean javoid_flush_during_recovery) { auto* opt = reinterpret_cast(jhandle); opt->avoid_flush_during_recovery = @@ -2151,88 +2151,88 @@ void Java_org_rocksdb_Options_setAvoidFlushDuringRecovery( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: avoidFlushDuringRecovery * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_avoidFlushDuringRecovery(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_avoidFlushDuringRecovery(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->avoid_flush_during_recovery); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAvoidUnnecessaryBlockingIO * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAvoidUnnecessaryBlockingIO( +void Java_org_forstdb_Options_setAvoidUnnecessaryBlockingIO( JNIEnv*, jclass, jlong jhandle, jboolean avoid_blocking_io) { auto* opt = reinterpret_cast(jhandle); opt->avoid_unnecessary_blocking_io = static_cast(avoid_blocking_io); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: avoidUnnecessaryBlockingIO * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_avoidUnnecessaryBlockingIO(JNIEnv*, jclass, +jboolean Java_org_forstdb_Options_avoidUnnecessaryBlockingIO(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->avoid_unnecessary_blocking_io); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setPersistStatsToDisk * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setPersistStatsToDisk( +void Java_org_forstdb_Options_setPersistStatsToDisk( JNIEnv*, jclass, jlong jhandle, jboolean persist_stats_to_disk) { auto* opt = reinterpret_cast(jhandle); opt->persist_stats_to_disk = static_cast(persist_stats_to_disk); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: persistStatsToDisk * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_persistStatsToDisk(JNIEnv*, jclass, +jboolean Java_org_forstdb_Options_persistStatsToDisk(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->persist_stats_to_disk); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setWriteDbidToManifest * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setWriteDbidToManifest( +void Java_org_forstdb_Options_setWriteDbidToManifest( JNIEnv*, jclass, jlong jhandle, jboolean jwrite_dbid_to_manifest) { auto* opt = reinterpret_cast(jhandle); opt->write_dbid_to_manifest = static_cast(jwrite_dbid_to_manifest); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: writeDbidToManifest * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_writeDbidToManifest(JNIEnv*, jclass, +jboolean Java_org_forstdb_Options_writeDbidToManifest(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->write_dbid_to_manifest); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLogReadaheadSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setLogReadaheadSize(JNIEnv*, jclass, +void Java_org_forstdb_Options_setLogReadaheadSize(JNIEnv*, jclass, jlong jhandle, jlong jlog_readahead_size) { auto* opt = reinterpret_cast(jhandle); @@ -2240,66 +2240,66 @@ void Java_org_rocksdb_Options_setLogReadaheadSize(JNIEnv*, jclass, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: logReasaheadSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_logReadaheadSize(JNIEnv*, jclass, +jlong Java_org_forstdb_Options_logReadaheadSize(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->log_readahead_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBestEffortsRecovery * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setBestEffortsRecovery( +void Java_org_forstdb_Options_setBestEffortsRecovery( JNIEnv*, jclass, jlong jhandle, jboolean jbest_efforts_recovery) { auto* opt = reinterpret_cast(jhandle); opt->best_efforts_recovery = static_cast(jbest_efforts_recovery); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: bestEffortsRecovery * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_bestEffortsRecovery(JNIEnv*, jclass, +jboolean Java_org_forstdb_Options_bestEffortsRecovery(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->best_efforts_recovery); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxBgErrorResumeCount * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxBgErrorResumeCount( +void Java_org_forstdb_Options_setMaxBgErrorResumeCount( JNIEnv*, jclass, jlong jhandle, jint jmax_bgerror_resume_count) { auto* opt = reinterpret_cast(jhandle); opt->max_bgerror_resume_count = static_cast(jmax_bgerror_resume_count); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxBgerrorResumeCount * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxBgerrorResumeCount(JNIEnv*, jclass, +jint Java_org_forstdb_Options_maxBgerrorResumeCount(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_bgerror_resume_count); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBgerrorResumeRetryInterval * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setBgerrorResumeRetryInterval( +void Java_org_forstdb_Options_setBgerrorResumeRetryInterval( JNIEnv*, jclass, jlong jhandle, jlong jbgerror_resume_retry_interval) { auto* opt = reinterpret_cast(jhandle); opt->bgerror_resume_retry_interval = @@ -2307,22 +2307,22 @@ void Java_org_rocksdb_Options_setBgerrorResumeRetryInterval( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: bgerrorResumeRetryInterval * Signature: (J)J */ -jlong Java_org_rocksdb_Options_bgerrorResumeRetryInterval(JNIEnv*, jclass, +jlong Java_org_forstdb_Options_bgerrorResumeRetryInterval(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->bgerror_resume_retry_interval); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAvoidFlushDuringShutdown * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAvoidFlushDuringShutdown( +void Java_org_forstdb_Options_setAvoidFlushDuringShutdown( JNIEnv*, jobject, jlong jhandle, jboolean javoid_flush_during_shutdown) { auto* opt = reinterpret_cast(jhandle); opt->avoid_flush_during_shutdown = @@ -2330,99 +2330,99 @@ void Java_org_rocksdb_Options_setAvoidFlushDuringShutdown( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: avoidFlushDuringShutdown * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_avoidFlushDuringShutdown(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_avoidFlushDuringShutdown(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->avoid_flush_during_shutdown); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAllowIngestBehind * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAllowIngestBehind( +void Java_org_forstdb_Options_setAllowIngestBehind( JNIEnv*, jobject, jlong jhandle, jboolean jallow_ingest_behind) { auto* opt = reinterpret_cast(jhandle); opt->allow_ingest_behind = jallow_ingest_behind == JNI_TRUE; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: allowIngestBehind * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_allowIngestBehind(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_allowIngestBehind(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->allow_ingest_behind); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setTwoWriteQueues * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setTwoWriteQueues(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setTwoWriteQueues(JNIEnv*, jobject, jlong jhandle, jboolean jtwo_write_queues) { auto* opt = reinterpret_cast(jhandle); opt->two_write_queues = jtwo_write_queues == JNI_TRUE; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: twoWriteQueues * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_twoWriteQueues(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_twoWriteQueues(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->two_write_queues); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setManualWalFlush * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setManualWalFlush(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setManualWalFlush(JNIEnv*, jobject, jlong jhandle, jboolean jmanual_wal_flush) { auto* opt = reinterpret_cast(jhandle); opt->manual_wal_flush = jmanual_wal_flush == JNI_TRUE; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: manualWalFlush * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_manualWalFlush(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_manualWalFlush(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->manual_wal_flush); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setAtomicFlush * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setAtomicFlush(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setAtomicFlush(JNIEnv*, jobject, jlong jhandle, jboolean jatomic_flush) { auto* opt = reinterpret_cast(jhandle); opt->atomic_flush = jatomic_flush == JNI_TRUE; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: atomicFlush * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_atomicFlush(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_Options_atomicFlush(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->atomic_flush); } @@ -2431,7 +2431,7 @@ jboolean Java_org_rocksdb_Options_atomicFlush(JNIEnv*, jobject, jlong jhandle) { * Method: tableFactoryName * Signature: (J)Ljava/lang/String */ -jstring Java_org_rocksdb_Options_tableFactoryName(JNIEnv* env, jobject, +jstring Java_org_forstdb_Options_tableFactoryName(JNIEnv* env, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); ROCKSDB_NAMESPACE::TableFactory* tf = opt->table_factory.get(); @@ -2444,44 +2444,44 @@ jstring Java_org_rocksdb_Options_tableFactoryName(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: minWriteBufferNumberToMerge * Signature: (J)I */ -jint Java_org_rocksdb_Options_minWriteBufferNumberToMerge(JNIEnv*, jobject, +jint Java_org_forstdb_Options_minWriteBufferNumberToMerge(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->min_write_buffer_number_to_merge; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMinWriteBufferNumberToMerge * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMinWriteBufferNumberToMerge( +void Java_org_forstdb_Options_setMinWriteBufferNumberToMerge( JNIEnv*, jobject, jlong jhandle, jint jmin_write_buffer_number_to_merge) { reinterpret_cast(jhandle) ->min_write_buffer_number_to_merge = static_cast(jmin_write_buffer_number_to_merge); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxWriteBufferNumberToMaintain * Signature: (J)I */ -jint Java_org_rocksdb_Options_maxWriteBufferNumberToMaintain(JNIEnv*, jobject, +jint Java_org_forstdb_Options_maxWriteBufferNumberToMaintain(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_write_buffer_number_to_maintain; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxWriteBufferNumberToMaintain * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxWriteBufferNumberToMaintain( +void Java_org_forstdb_Options_setMaxWriteBufferNumberToMaintain( JNIEnv*, jobject, jlong jhandle, jint jmax_write_buffer_number_to_maintain) { reinterpret_cast(jhandle) @@ -2490,11 +2490,11 @@ void Java_org_rocksdb_Options_setMaxWriteBufferNumberToMaintain( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompressionType * Signature: (JB)V */ -void Java_org_rocksdb_Options_setCompressionType( +void Java_org_forstdb_Options_setCompressionType( JNIEnv*, jobject, jlong jhandle, jbyte jcompression_type_value) { auto* opts = reinterpret_cast(jhandle); opts->compression = @@ -2503,11 +2503,11 @@ void Java_org_rocksdb_Options_setCompressionType( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: compressionType * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_compressionType(JNIEnv*, jobject, +jbyte Java_org_forstdb_Options_compressionType(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( @@ -2594,11 +2594,11 @@ jbyteArray rocksdb_compression_list_helper( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompressionPerLevel * Signature: (J[B)V */ -void Java_org_rocksdb_Options_setCompressionPerLevel( +void Java_org_forstdb_Options_setCompressionPerLevel( JNIEnv* env, jobject, jlong jhandle, jbyteArray jcompressionLevels) { auto uptr_compression_levels = rocksdb_compression_vector_helper(env, jcompressionLevels); @@ -2611,22 +2611,22 @@ void Java_org_rocksdb_Options_setCompressionPerLevel( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: compressionPerLevel * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_Options_compressionPerLevel(JNIEnv* env, jobject, +jbyteArray Java_org_forstdb_Options_compressionPerLevel(JNIEnv* env, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); return rocksdb_compression_list_helper(env, options->compression_per_level); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBottommostCompressionType * Signature: (JB)V */ -void Java_org_rocksdb_Options_setBottommostCompressionType( +void Java_org_forstdb_Options_setBottommostCompressionType( JNIEnv*, jobject, jlong jhandle, jbyte jcompression_type_value) { auto* options = reinterpret_cast(jhandle); options->bottommost_compression = @@ -2635,11 +2635,11 @@ void Java_org_rocksdb_Options_setBottommostCompressionType( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: bottommostCompressionType * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_bottommostCompressionType(JNIEnv*, jobject, +jbyte Java_org_forstdb_Options_bottommostCompressionType(JNIEnv*, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( @@ -2647,11 +2647,11 @@ jbyte Java_org_rocksdb_Options_bottommostCompressionType(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBottommostCompressionOptions * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setBottommostCompressionOptions( +void Java_org_forstdb_Options_setBottommostCompressionOptions( JNIEnv*, jobject, jlong jhandle, jlong jbottommost_compression_options_handle) { auto* options = reinterpret_cast(jhandle); @@ -2662,11 +2662,11 @@ void Java_org_rocksdb_Options_setBottommostCompressionOptions( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompressionOptions * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setCompressionOptions( +void Java_org_forstdb_Options_setCompressionOptions( JNIEnv*, jobject, jlong jhandle, jlong jcompression_options_handle) { auto* options = reinterpret_cast(jhandle); auto* compression_options = @@ -2676,11 +2676,11 @@ void Java_org_rocksdb_Options_setCompressionOptions( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompactionStyle * Signature: (JB)V */ -void Java_org_rocksdb_Options_setCompactionStyle(JNIEnv*, jobject, +void Java_org_forstdb_Options_setCompactionStyle(JNIEnv*, jobject, jlong jhandle, jbyte jcompaction_style) { auto* options = reinterpret_cast(jhandle); @@ -2690,11 +2690,11 @@ void Java_org_rocksdb_Options_setCompactionStyle(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: compactionStyle * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_compactionStyle(JNIEnv*, jobject, +jbyte Java_org_forstdb_Options_compactionStyle(JNIEnv*, jobject, jlong jhandle) { auto* options = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::CompactionStyleJni::toJavaCompactionStyle( @@ -2702,11 +2702,11 @@ jbyte Java_org_rocksdb_Options_compactionStyle(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxTableFilesSizeFIFO * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMaxTableFilesSizeFIFO( +void Java_org_forstdb_Options_setMaxTableFilesSizeFIFO( JNIEnv*, jobject, jlong jhandle, jlong jmax_table_files_size) { reinterpret_cast(jhandle) ->compaction_options_fifo.max_table_files_size = @@ -2714,42 +2714,42 @@ void Java_org_rocksdb_Options_setMaxTableFilesSizeFIFO( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxTableFilesSizeFIFO * Signature: (J)J */ -jlong Java_org_rocksdb_Options_maxTableFilesSizeFIFO(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_maxTableFilesSizeFIFO(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->compaction_options_fifo.max_table_files_size; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: numLevels * Signature: (J)I */ -jint Java_org_rocksdb_Options_numLevels(JNIEnv*, jobject, jlong jhandle) { +jint Java_org_forstdb_Options_numLevels(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->num_levels; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setNumLevels * Signature: (JI)V */ -void Java_org_rocksdb_Options_setNumLevels(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setNumLevels(JNIEnv*, jobject, jlong jhandle, jint jnum_levels) { reinterpret_cast(jhandle)->num_levels = static_cast(jnum_levels); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: levelZeroFileNumCompactionTrigger * Signature: (J)I */ -jint Java_org_rocksdb_Options_levelZeroFileNumCompactionTrigger(JNIEnv*, +jint Java_org_forstdb_Options_levelZeroFileNumCompactionTrigger(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) @@ -2757,11 +2757,11 @@ jint Java_org_rocksdb_Options_levelZeroFileNumCompactionTrigger(JNIEnv*, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLevelZeroFileNumCompactionTrigger * Signature: (JI)V */ -void Java_org_rocksdb_Options_setLevelZeroFileNumCompactionTrigger( +void Java_org_forstdb_Options_setLevelZeroFileNumCompactionTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_file_num_compaction_trigger) { reinterpret_cast(jhandle) ->level0_file_num_compaction_trigger = @@ -2769,22 +2769,22 @@ void Java_org_rocksdb_Options_setLevelZeroFileNumCompactionTrigger( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: levelZeroSlowdownWritesTrigger * Signature: (J)I */ -jint Java_org_rocksdb_Options_levelZeroSlowdownWritesTrigger(JNIEnv*, jobject, +jint Java_org_forstdb_Options_levelZeroSlowdownWritesTrigger(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_slowdown_writes_trigger; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLevelSlowdownWritesTrigger * Signature: (JI)V */ -void Java_org_rocksdb_Options_setLevelZeroSlowdownWritesTrigger( +void Java_org_forstdb_Options_setLevelZeroSlowdownWritesTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_slowdown_writes_trigger) { reinterpret_cast(jhandle) ->level0_slowdown_writes_trigger = @@ -2792,22 +2792,22 @@ void Java_org_rocksdb_Options_setLevelZeroSlowdownWritesTrigger( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: levelZeroStopWritesTrigger * Signature: (J)I */ -jint Java_org_rocksdb_Options_levelZeroStopWritesTrigger(JNIEnv*, jobject, +jint Java_org_forstdb_Options_levelZeroStopWritesTrigger(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_stop_writes_trigger; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLevelStopWritesTrigger * Signature: (JI)V */ -void Java_org_rocksdb_Options_setLevelZeroStopWritesTrigger( +void Java_org_forstdb_Options_setLevelZeroStopWritesTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_stop_writes_trigger) { reinterpret_cast(jhandle) ->level0_stop_writes_trigger = @@ -2815,44 +2815,44 @@ void Java_org_rocksdb_Options_setLevelZeroStopWritesTrigger( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: targetFileSizeBase * Signature: (J)J */ -jlong Java_org_rocksdb_Options_targetFileSizeBase(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_targetFileSizeBase(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->target_file_size_base; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setTargetFileSizeBase * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setTargetFileSizeBase( +void Java_org_forstdb_Options_setTargetFileSizeBase( JNIEnv*, jobject, jlong jhandle, jlong jtarget_file_size_base) { reinterpret_cast(jhandle) ->target_file_size_base = static_cast(jtarget_file_size_base); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: targetFileSizeMultiplier * Signature: (J)I */ -jint Java_org_rocksdb_Options_targetFileSizeMultiplier(JNIEnv*, jobject, +jint Java_org_forstdb_Options_targetFileSizeMultiplier(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->target_file_size_multiplier; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setTargetFileSizeMultiplier * Signature: (JI)V */ -void Java_org_rocksdb_Options_setTargetFileSizeMultiplier( +void Java_org_forstdb_Options_setTargetFileSizeMultiplier( JNIEnv*, jobject, jlong jhandle, jint jtarget_file_size_multiplier) { reinterpret_cast(jhandle) ->target_file_size_multiplier = @@ -2860,22 +2860,22 @@ void Java_org_rocksdb_Options_setTargetFileSizeMultiplier( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxBytesForLevelBase * Signature: (J)J */ -jlong Java_org_rocksdb_Options_maxBytesForLevelBase(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_maxBytesForLevelBase(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_bytes_for_level_base; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxBytesForLevelBase * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMaxBytesForLevelBase( +void Java_org_forstdb_Options_setMaxBytesForLevelBase( JNIEnv*, jobject, jlong jhandle, jlong jmax_bytes_for_level_base) { reinterpret_cast(jhandle) ->max_bytes_for_level_base = @@ -2883,44 +2883,44 @@ void Java_org_rocksdb_Options_setMaxBytesForLevelBase( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: levelCompactionDynamicLevelBytes * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_levelCompactionDynamicLevelBytes( +jboolean Java_org_forstdb_Options_levelCompactionDynamicLevelBytes( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level_compaction_dynamic_level_bytes; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLevelCompactionDynamicLevelBytes * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setLevelCompactionDynamicLevelBytes( +void Java_org_forstdb_Options_setLevelCompactionDynamicLevelBytes( JNIEnv*, jobject, jlong jhandle, jboolean jenable_dynamic_level_bytes) { reinterpret_cast(jhandle) ->level_compaction_dynamic_level_bytes = (jenable_dynamic_level_bytes); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxBytesForLevelMultiplier * Signature: (J)D */ -jdouble Java_org_rocksdb_Options_maxBytesForLevelMultiplier(JNIEnv*, jobject, +jdouble Java_org_forstdb_Options_maxBytesForLevelMultiplier(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_bytes_for_level_multiplier; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxBytesForLevelMultiplier * Signature: (JD)V */ -void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplier( +void Java_org_forstdb_Options_setMaxBytesForLevelMultiplier( JNIEnv*, jobject, jlong jhandle, jdouble jmax_bytes_for_level_multiplier) { reinterpret_cast(jhandle) ->max_bytes_for_level_multiplier = @@ -2928,11 +2928,11 @@ void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplier( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxCompactionBytes * Signature: (J)I */ -jlong Java_org_rocksdb_Options_maxCompactionBytes(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_maxCompactionBytes(JNIEnv*, jobject, jlong jhandle) { return static_cast( reinterpret_cast(jhandle) @@ -2940,32 +2940,32 @@ jlong Java_org_rocksdb_Options_maxCompactionBytes(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxCompactionBytes * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMaxCompactionBytes( +void Java_org_forstdb_Options_setMaxCompactionBytes( JNIEnv*, jobject, jlong jhandle, jlong jmax_compaction_bytes) { reinterpret_cast(jhandle)->max_compaction_bytes = static_cast(jmax_compaction_bytes); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: arenaBlockSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_arenaBlockSize(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_arenaBlockSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->arena_block_size; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setArenaBlockSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setArenaBlockSize(JNIEnv* env, jobject, +void Java_org_forstdb_Options_setArenaBlockSize(JNIEnv* env, jobject, jlong jhandle, jlong jarena_block_size) { auto s = @@ -2979,44 +2979,44 @@ void Java_org_rocksdb_Options_setArenaBlockSize(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: disableAutoCompactions * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_disableAutoCompactions(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_disableAutoCompactions(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->disable_auto_compactions; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setDisableAutoCompactions * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setDisableAutoCompactions( +void Java_org_forstdb_Options_setDisableAutoCompactions( JNIEnv*, jobject, jlong jhandle, jboolean jdisable_auto_compactions) { reinterpret_cast(jhandle) ->disable_auto_compactions = static_cast(jdisable_auto_compactions); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxSequentialSkipInIterations * Signature: (J)J */ -jlong Java_org_rocksdb_Options_maxSequentialSkipInIterations(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_maxSequentialSkipInIterations(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_sequential_skip_in_iterations; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxSequentialSkipInIterations * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMaxSequentialSkipInIterations( +void Java_org_forstdb_Options_setMaxSequentialSkipInIterations( JNIEnv*, jobject, jlong jhandle, jlong jmax_sequential_skip_in_iterations) { reinterpret_cast(jhandle) ->max_sequential_skip_in_iterations = @@ -3024,44 +3024,44 @@ void Java_org_rocksdb_Options_setMaxSequentialSkipInIterations( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: inplaceUpdateSupport * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_inplaceUpdateSupport(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_inplaceUpdateSupport(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->inplace_update_support; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setInplaceUpdateSupport * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setInplaceUpdateSupport( +void Java_org_forstdb_Options_setInplaceUpdateSupport( JNIEnv*, jobject, jlong jhandle, jboolean jinplace_update_support) { reinterpret_cast(jhandle) ->inplace_update_support = static_cast(jinplace_update_support); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: inplaceUpdateNumLocks * Signature: (J)J */ -jlong Java_org_rocksdb_Options_inplaceUpdateNumLocks(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_inplaceUpdateNumLocks(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->inplace_update_num_locks; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setInplaceUpdateNumLocks * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setInplaceUpdateNumLocks( +void Java_org_forstdb_Options_setInplaceUpdateNumLocks( JNIEnv* env, jobject, jlong jhandle, jlong jinplace_update_num_locks) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( jinplace_update_num_locks); @@ -3074,22 +3074,22 @@ void Java_org_rocksdb_Options_setInplaceUpdateNumLocks( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: memtablePrefixBloomSizeRatio * Signature: (J)I */ -jdouble Java_org_rocksdb_Options_memtablePrefixBloomSizeRatio(JNIEnv*, jobject, +jdouble Java_org_forstdb_Options_memtablePrefixBloomSizeRatio(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->memtable_prefix_bloom_size_ratio; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMemtablePrefixBloomSizeRatio * Signature: (JI)V */ -void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio( +void Java_org_forstdb_Options_setMemtablePrefixBloomSizeRatio( JNIEnv*, jobject, jlong jhandle, jdouble jmemtable_prefix_bloom_size_ratio) { reinterpret_cast(jhandle) @@ -3098,22 +3098,22 @@ void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: experimentalMempurgeThreshold * Signature: (J)I */ -jdouble Java_org_rocksdb_Options_experimentalMempurgeThreshold(JNIEnv*, jobject, +jdouble Java_org_forstdb_Options_experimentalMempurgeThreshold(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->experimental_mempurge_threshold; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setExperimentalMempurgeThreshold * Signature: (JI)V */ -void Java_org_rocksdb_Options_setExperimentalMempurgeThreshold( +void Java_org_forstdb_Options_setExperimentalMempurgeThreshold( JNIEnv*, jobject, jlong jhandle, jdouble jexperimental_mempurge_threshold) { reinterpret_cast(jhandle) ->experimental_mempurge_threshold = @@ -3121,22 +3121,22 @@ void Java_org_rocksdb_Options_setExperimentalMempurgeThreshold( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: memtableWholeKeyFiltering * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_memtableWholeKeyFiltering(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_memtableWholeKeyFiltering(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->memtable_whole_key_filtering; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMemtableWholeKeyFiltering * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setMemtableWholeKeyFiltering( +void Java_org_forstdb_Options_setMemtableWholeKeyFiltering( JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) { reinterpret_cast(jhandle) ->memtable_whole_key_filtering = @@ -3144,42 +3144,42 @@ void Java_org_rocksdb_Options_setMemtableWholeKeyFiltering( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: bloomLocality * Signature: (J)I */ -jint Java_org_rocksdb_Options_bloomLocality(JNIEnv*, jobject, jlong jhandle) { +jint Java_org_forstdb_Options_bloomLocality(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->bloom_locality; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBloomLocality * Signature: (JI)V */ -void Java_org_rocksdb_Options_setBloomLocality(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setBloomLocality(JNIEnv*, jobject, jlong jhandle, jint jbloom_locality) { reinterpret_cast(jhandle)->bloom_locality = static_cast(jbloom_locality); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxSuccessiveMerges * Signature: (J)J */ -jlong Java_org_rocksdb_Options_maxSuccessiveMerges(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_maxSuccessiveMerges(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_successive_merges; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxSuccessiveMerges * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMaxSuccessiveMerges( +void Java_org_forstdb_Options_setMaxSuccessiveMerges( JNIEnv* env, jobject, jlong jhandle, jlong jmax_successive_merges) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( jmax_successive_merges); @@ -3192,22 +3192,22 @@ void Java_org_rocksdb_Options_setMaxSuccessiveMerges( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: optimizeFiltersForHits * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_optimizeFiltersForHits(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_optimizeFiltersForHits(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->optimize_filters_for_hits; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setOptimizeFiltersForHits * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setOptimizeFiltersForHits( +void Java_org_forstdb_Options_setOptimizeFiltersForHits( JNIEnv*, jobject, jlong jhandle, jboolean joptimize_filters_for_hits) { reinterpret_cast(jhandle) ->optimize_filters_for_hits = @@ -3215,11 +3215,11 @@ void Java_org_rocksdb_Options_setOptimizeFiltersForHits( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: oldDefaults * Signature: (JII)V */ -void Java_org_rocksdb_Options_oldDefaults(JNIEnv*, jclass, jlong jhandle, +void Java_org_forstdb_Options_oldDefaults(JNIEnv*, jclass, jlong jhandle, jint major_version, jint minor_version) { reinterpret_cast(jhandle)->OldDefaults( @@ -3227,21 +3227,21 @@ void Java_org_rocksdb_Options_oldDefaults(JNIEnv*, jclass, jlong jhandle, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: optimizeForSmallDb * Signature: (J)V */ -void Java_org_rocksdb_Options_optimizeForSmallDb__J(JNIEnv*, jobject, +void Java_org_forstdb_Options_optimizeForSmallDb__J(JNIEnv*, jobject, jlong jhandle) { reinterpret_cast(jhandle)->OptimizeForSmallDb(); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: optimizeForSmallDb * Signature: (JJ)V */ -void Java_org_rocksdb_Options_optimizeForSmallDb__JJ(JNIEnv*, jclass, +void Java_org_forstdb_Options_optimizeForSmallDb__JJ(JNIEnv*, jclass, jlong jhandle, jlong cache_handle) { auto* cache_sptr_ptr = @@ -3254,65 +3254,65 @@ void Java_org_rocksdb_Options_optimizeForSmallDb__JJ(JNIEnv*, jclass, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: optimizeForPointLookup * Signature: (JJ)V */ -void Java_org_rocksdb_Options_optimizeForPointLookup( +void Java_org_forstdb_Options_optimizeForPointLookup( JNIEnv*, jobject, jlong jhandle, jlong block_cache_size_mb) { reinterpret_cast(jhandle) ->OptimizeForPointLookup(block_cache_size_mb); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: optimizeLevelStyleCompaction * Signature: (JJ)V */ -void Java_org_rocksdb_Options_optimizeLevelStyleCompaction( +void Java_org_forstdb_Options_optimizeLevelStyleCompaction( JNIEnv*, jobject, jlong jhandle, jlong memtable_memory_budget) { reinterpret_cast(jhandle) ->OptimizeLevelStyleCompaction(memtable_memory_budget); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: optimizeUniversalStyleCompaction * Signature: (JJ)V */ -void Java_org_rocksdb_Options_optimizeUniversalStyleCompaction( +void Java_org_forstdb_Options_optimizeUniversalStyleCompaction( JNIEnv*, jobject, jlong jhandle, jlong memtable_memory_budget) { reinterpret_cast(jhandle) ->OptimizeUniversalStyleCompaction(memtable_memory_budget); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: prepareForBulkLoad * Signature: (J)V */ -void Java_org_rocksdb_Options_prepareForBulkLoad(JNIEnv*, jobject, +void Java_org_forstdb_Options_prepareForBulkLoad(JNIEnv*, jobject, jlong jhandle) { reinterpret_cast(jhandle)->PrepareForBulkLoad(); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: memtableHugePageSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_memtableHugePageSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_memtableHugePageSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->memtable_huge_page_size; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMemtableHugePageSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMemtableHugePageSize( +void Java_org_forstdb_Options_setMemtableHugePageSize( JNIEnv* env, jobject, jlong jhandle, jlong jmemtable_huge_page_size) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( jmemtable_huge_page_size); @@ -3325,22 +3325,22 @@ void Java_org_rocksdb_Options_setMemtableHugePageSize( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: softPendingCompactionBytesLimit * Signature: (J)J */ -jlong Java_org_rocksdb_Options_softPendingCompactionBytesLimit(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_softPendingCompactionBytesLimit(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->soft_pending_compaction_bytes_limit; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setSoftPendingCompactionBytesLimit * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setSoftPendingCompactionBytesLimit( +void Java_org_forstdb_Options_setSoftPendingCompactionBytesLimit( JNIEnv*, jobject, jlong jhandle, jlong jsoft_pending_compaction_bytes_limit) { reinterpret_cast(jhandle) @@ -3349,22 +3349,22 @@ void Java_org_rocksdb_Options_setSoftPendingCompactionBytesLimit( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: softHardCompactionBytesLimit * Signature: (J)J */ -jlong Java_org_rocksdb_Options_hardPendingCompactionBytesLimit(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_hardPendingCompactionBytesLimit(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->hard_pending_compaction_bytes_limit; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setHardPendingCompactionBytesLimit * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setHardPendingCompactionBytesLimit( +void Java_org_forstdb_Options_setHardPendingCompactionBytesLimit( JNIEnv*, jobject, jlong jhandle, jlong jhard_pending_compaction_bytes_limit) { reinterpret_cast(jhandle) @@ -3373,22 +3373,22 @@ void Java_org_rocksdb_Options_setHardPendingCompactionBytesLimit( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: level0FileNumCompactionTrigger * Signature: (J)I */ -jint Java_org_rocksdb_Options_level0FileNumCompactionTrigger(JNIEnv*, jobject, +jint Java_org_forstdb_Options_level0FileNumCompactionTrigger(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_file_num_compaction_trigger; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLevel0FileNumCompactionTrigger * Signature: (JI)V */ -void Java_org_rocksdb_Options_setLevel0FileNumCompactionTrigger( +void Java_org_forstdb_Options_setLevel0FileNumCompactionTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_file_num_compaction_trigger) { reinterpret_cast(jhandle) ->level0_file_num_compaction_trigger = @@ -3396,22 +3396,22 @@ void Java_org_rocksdb_Options_setLevel0FileNumCompactionTrigger( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: level0SlowdownWritesTrigger * Signature: (J)I */ -jint Java_org_rocksdb_Options_level0SlowdownWritesTrigger(JNIEnv*, jobject, +jint Java_org_forstdb_Options_level0SlowdownWritesTrigger(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_slowdown_writes_trigger; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLevel0SlowdownWritesTrigger * Signature: (JI)V */ -void Java_org_rocksdb_Options_setLevel0SlowdownWritesTrigger( +void Java_org_forstdb_Options_setLevel0SlowdownWritesTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_slowdown_writes_trigger) { reinterpret_cast(jhandle) ->level0_slowdown_writes_trigger = @@ -3419,22 +3419,22 @@ void Java_org_rocksdb_Options_setLevel0SlowdownWritesTrigger( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: level0StopWritesTrigger * Signature: (J)I */ -jint Java_org_rocksdb_Options_level0StopWritesTrigger(JNIEnv*, jobject, +jint Java_org_forstdb_Options_level0StopWritesTrigger(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_stop_writes_trigger; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setLevel0StopWritesTrigger * Signature: (JI)V */ -void Java_org_rocksdb_Options_setLevel0StopWritesTrigger( +void Java_org_forstdb_Options_setLevel0StopWritesTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_stop_writes_trigger) { reinterpret_cast(jhandle) ->level0_stop_writes_trigger = @@ -3442,11 +3442,11 @@ void Java_org_rocksdb_Options_setLevel0StopWritesTrigger( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: maxBytesForLevelMultiplierAdditional * Signature: (J)[I */ -jintArray Java_org_rocksdb_Options_maxBytesForLevelMultiplierAdditional( +jintArray Java_org_forstdb_Options_maxBytesForLevelMultiplierAdditional( JNIEnv* env, jobject, jlong jhandle) { auto mbflma = reinterpret_cast(jhandle) ->max_bytes_for_level_multiplier_additional; @@ -3480,11 +3480,11 @@ jintArray Java_org_rocksdb_Options_maxBytesForLevelMultiplierAdditional( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMaxBytesForLevelMultiplierAdditional * Signature: (J[I)V */ -void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplierAdditional( +void Java_org_forstdb_Options_setMaxBytesForLevelMultiplierAdditional( JNIEnv* env, jobject, jlong jhandle, jintArray jmax_bytes_for_level_multiplier_additional) { jsize len = env->GetArrayLength(jmax_bytes_for_level_multiplier_additional); @@ -3507,33 +3507,33 @@ void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplierAdditional( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: paranoidFileChecks * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_paranoidFileChecks(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_paranoidFileChecks(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->paranoid_file_checks; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setParanoidFileChecks * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setParanoidFileChecks( +void Java_org_forstdb_Options_setParanoidFileChecks( JNIEnv*, jobject, jlong jhandle, jboolean jparanoid_file_checks) { reinterpret_cast(jhandle)->paranoid_file_checks = static_cast(jparanoid_file_checks); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompactionPriority * Signature: (JB)V */ -void Java_org_rocksdb_Options_setCompactionPriority( +void Java_org_forstdb_Options_setCompactionPriority( JNIEnv*, jobject, jlong jhandle, jbyte jcompaction_priority_value) { auto* opts = reinterpret_cast(jhandle); opts->compaction_pri = @@ -3542,11 +3542,11 @@ void Java_org_rocksdb_Options_setCompactionPriority( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: compactionPriority * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_compactionPriority(JNIEnv*, jobject, +jbyte Java_org_forstdb_Options_compactionPriority(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::CompactionPriorityJni::toJavaCompactionPriority( @@ -3554,11 +3554,11 @@ jbyte Java_org_rocksdb_Options_compactionPriority(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setReportBgIoStats * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setReportBgIoStats(JNIEnv*, jobject, +void Java_org_forstdb_Options_setReportBgIoStats(JNIEnv*, jobject, jlong jhandle, jboolean jreport_bg_io_stats) { auto* opts = reinterpret_cast(jhandle); @@ -3566,43 +3566,43 @@ void Java_org_rocksdb_Options_setReportBgIoStats(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: reportBgIoStats * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_reportBgIoStats(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_reportBgIoStats(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->report_bg_io_stats); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setTtl * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setTtl(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setTtl(JNIEnv*, jobject, jlong jhandle, jlong jttl) { auto* opts = reinterpret_cast(jhandle); opts->ttl = static_cast(jttl); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: ttl * Signature: (J)J */ -jlong Java_org_rocksdb_Options_ttl(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_ttl(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->ttl); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setPeriodicCompactionSeconds * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setPeriodicCompactionSeconds( +void Java_org_forstdb_Options_setPeriodicCompactionSeconds( JNIEnv*, jobject, jlong jhandle, jlong jperiodicCompactionSeconds) { auto* opts = reinterpret_cast(jhandle); opts->periodic_compaction_seconds = @@ -3610,22 +3610,22 @@ void Java_org_rocksdb_Options_setPeriodicCompactionSeconds( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: periodicCompactionSeconds * Signature: (J)J */ -jlong Java_org_rocksdb_Options_periodicCompactionSeconds(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_periodicCompactionSeconds(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->periodic_compaction_seconds); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompactionOptionsUniversal * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setCompactionOptionsUniversal( +void Java_org_forstdb_Options_setCompactionOptionsUniversal( JNIEnv*, jobject, jlong jhandle, jlong jcompaction_options_universal_handle) { auto* opts = reinterpret_cast(jhandle); @@ -3636,11 +3636,11 @@ void Java_org_rocksdb_Options_setCompactionOptionsUniversal( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setCompactionOptionsFIFO * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setCompactionOptionsFIFO( +void Java_org_forstdb_Options_setCompactionOptionsFIFO( JNIEnv*, jobject, jlong jhandle, jlong jcompaction_options_fifo_handle) { auto* opts = reinterpret_cast(jhandle); auto* opts_fifo = reinterpret_cast( @@ -3649,22 +3649,22 @@ void Java_org_rocksdb_Options_setCompactionOptionsFIFO( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setForceConsistencyChecks * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setForceConsistencyChecks( +void Java_org_forstdb_Options_setForceConsistencyChecks( JNIEnv*, jobject, jlong jhandle, jboolean jforce_consistency_checks) { auto* opts = reinterpret_cast(jhandle); opts->force_consistency_checks = static_cast(jforce_consistency_checks); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: forceConsistencyChecks * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_forceConsistencyChecks(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_forceConsistencyChecks(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->force_consistency_checks); @@ -3673,11 +3673,11 @@ jboolean Java_org_rocksdb_Options_forceConsistencyChecks(JNIEnv*, jobject, /// BLOB options /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setEnableBlobFiles * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setEnableBlobFiles(JNIEnv*, jobject, +void Java_org_forstdb_Options_setEnableBlobFiles(JNIEnv*, jobject, jlong jhandle, jboolean jenable_blob_files) { auto* opts = reinterpret_cast(jhandle); @@ -3685,64 +3685,64 @@ void Java_org_rocksdb_Options_setEnableBlobFiles(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: enableBlobFiles * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_enableBlobFiles(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_enableBlobFiles(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->enable_blob_files); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setMinBlobSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setMinBlobSize(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setMinBlobSize(JNIEnv*, jobject, jlong jhandle, jlong jmin_blob_size) { auto* opts = reinterpret_cast(jhandle); opts->min_blob_size = static_cast(jmin_blob_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: minBlobSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_minBlobSize(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_minBlobSize(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->min_blob_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBlobFileSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setBlobFileSize(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Options_setBlobFileSize(JNIEnv*, jobject, jlong jhandle, jlong jblob_file_size) { auto* opts = reinterpret_cast(jhandle); opts->blob_file_size = static_cast(jblob_file_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: blobFileSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_blobFileSize(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_Options_blobFileSize(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->blob_file_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBlobCompressionType * Signature: (JB)V */ -void Java_org_rocksdb_Options_setBlobCompressionType( +void Java_org_forstdb_Options_setBlobCompressionType( JNIEnv*, jobject, jlong jhandle, jbyte jblob_compression_type_value) { auto* opts = reinterpret_cast(jhandle); opts->blob_compression_type = @@ -3751,11 +3751,11 @@ void Java_org_rocksdb_Options_setBlobCompressionType( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: blobCompressionType * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_blobCompressionType(JNIEnv*, jobject, +jbyte Java_org_forstdb_Options_blobCompressionType(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( @@ -3763,11 +3763,11 @@ jbyte Java_org_rocksdb_Options_blobCompressionType(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setEnableBlobGarbageCollection * Signature: (JZ)V */ -void Java_org_rocksdb_Options_setEnableBlobGarbageCollection( +void Java_org_forstdb_Options_setEnableBlobGarbageCollection( JNIEnv*, jobject, jlong jhandle, jboolean jenable_blob_garbage_collection) { auto* opts = reinterpret_cast(jhandle); opts->enable_blob_garbage_collection = @@ -3775,22 +3775,22 @@ void Java_org_rocksdb_Options_setEnableBlobGarbageCollection( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: enableBlobGarbageCollection * Signature: (J)Z */ -jboolean Java_org_rocksdb_Options_enableBlobGarbageCollection(JNIEnv*, jobject, +jboolean Java_org_forstdb_Options_enableBlobGarbageCollection(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->enable_blob_garbage_collection); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBlobGarbageCollectionAgeCutoff * Signature: (JD)V */ -void Java_org_rocksdb_Options_setBlobGarbageCollectionAgeCutoff( +void Java_org_forstdb_Options_setBlobGarbageCollectionAgeCutoff( JNIEnv*, jobject, jlong jhandle, jdouble jblob_garbage_collection_age_cutoff) { auto* opts = reinterpret_cast(jhandle); @@ -3799,11 +3799,11 @@ void Java_org_rocksdb_Options_setBlobGarbageCollectionAgeCutoff( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: blobGarbageCollectionAgeCutoff * Signature: (J)D */ -jdouble Java_org_rocksdb_Options_blobGarbageCollectionAgeCutoff(JNIEnv*, +jdouble Java_org_forstdb_Options_blobGarbageCollectionAgeCutoff(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -3811,11 +3811,11 @@ jdouble Java_org_rocksdb_Options_blobGarbageCollectionAgeCutoff(JNIEnv*, } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBlobGarbageCollectionForceThreshold * Signature: (JD)V */ -void Java_org_rocksdb_Options_setBlobGarbageCollectionForceThreshold( +void Java_org_forstdb_Options_setBlobGarbageCollectionForceThreshold( JNIEnv*, jobject, jlong jhandle, jdouble jblob_garbage_collection_force_threshold) { auto* opts = reinterpret_cast(jhandle); @@ -3824,22 +3824,22 @@ void Java_org_rocksdb_Options_setBlobGarbageCollectionForceThreshold( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: blobGarbageCollectionForceThreshold * Signature: (J)D */ -jdouble Java_org_rocksdb_Options_blobGarbageCollectionForceThreshold( +jdouble Java_org_forstdb_Options_blobGarbageCollectionForceThreshold( JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->blob_garbage_collection_force_threshold); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBlobCompactionReadaheadSize * Signature: (JJ)V */ -void Java_org_rocksdb_Options_setBlobCompactionReadaheadSize( +void Java_org_forstdb_Options_setBlobCompactionReadaheadSize( JNIEnv*, jobject, jlong jhandle, jlong jblob_compaction_readahead_size) { auto* opts = reinterpret_cast(jhandle); opts->blob_compaction_readahead_size = @@ -3847,44 +3847,44 @@ void Java_org_rocksdb_Options_setBlobCompactionReadaheadSize( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: blobCompactionReadaheadSize * Signature: (J)J */ -jlong Java_org_rocksdb_Options_blobCompactionReadaheadSize(JNIEnv*, jobject, +jlong Java_org_forstdb_Options_blobCompactionReadaheadSize(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->blob_compaction_readahead_size); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setBlobFileStartingLevel * Signature: (JI)V */ -void Java_org_rocksdb_Options_setBlobFileStartingLevel( +void Java_org_forstdb_Options_setBlobFileStartingLevel( JNIEnv*, jobject, jlong jhandle, jint jblob_file_starting_level) { auto* opts = reinterpret_cast(jhandle); opts->blob_file_starting_level = jblob_file_starting_level; } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: blobFileStartingLevel * Signature: (J)I */ -jint Java_org_rocksdb_Options_blobFileStartingLevel(JNIEnv*, jobject, +jint Java_org_forstdb_Options_blobFileStartingLevel(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return static_cast(opts->blob_file_starting_level); } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: setPrepopulateBlobCache * Signature: (JB)V */ -void Java_org_rocksdb_Options_setPrepopulateBlobCache( +void Java_org_forstdb_Options_setPrepopulateBlobCache( JNIEnv*, jobject, jlong jhandle, jbyte jprepopulate_blob_cache_value) { auto* opts = reinterpret_cast(jhandle); opts->prepopulate_blob_cache = @@ -3893,37 +3893,60 @@ void Java_org_rocksdb_Options_setPrepopulateBlobCache( } /* - * Class: org_rocksdb_Options + * Class: org_forstdb_Options * Method: prepopulateBlobCache * Signature: (J)B */ -jbyte Java_org_rocksdb_Options_prepopulateBlobCache(JNIEnv*, jobject, +jbyte Java_org_forstdb_Options_prepopulateBlobCache(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::PrepopulateBlobCacheJni::toJavaPrepopulateBlobCache( opts->prepopulate_blob_cache); } +/* + * Class: org_forstdb_Options + * Method: setMemtableMaxRangeDeletions + * Signature: (JI)V + */ +void Java_org_forstdb_Options_setMemtableMaxRangeDeletions( + JNIEnv*, jobject, jlong jhandle, jint jmemtable_max_range_deletions) { + auto* opts = reinterpret_cast(jhandle); + opts->memtable_max_range_deletions = + static_cast(jmemtable_max_range_deletions); +} + +/* + * Class: org_forstdb_Options + * Method: memtableMaxRangeDeletions + * Signature: (J)I + */ +jint Java_org_forstdb_Options_memtableMaxRangeDeletions(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->memtable_max_range_deletions); +} + ////////////////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::ColumnFamilyOptions /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: newColumnFamilyOptions * Signature: ()J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptions(JNIEnv*, +jlong Java_org_forstdb_ColumnFamilyOptions_newColumnFamilyOptions(JNIEnv*, jclass) { auto* op = new ROCKSDB_NAMESPACE::ColumnFamilyOptions(); return GET_CPLUSPLUS_POINTER(op); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: copyColumnFamilyOptions * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_copyColumnFamilyOptions( +jlong Java_org_forstdb_ColumnFamilyOptions_copyColumnFamilyOptions( JNIEnv*, jclass, jlong jhandle) { auto new_opt = new ROCKSDB_NAMESPACE::ColumnFamilyOptions( *(reinterpret_cast(jhandle))); @@ -3931,11 +3954,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_copyColumnFamilyOptions( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: newColumnFamilyOptionsFromOptions * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptionsFromOptions( +jlong Java_org_forstdb_ColumnFamilyOptions_newColumnFamilyOptionsFromOptions( JNIEnv*, jclass, jlong joptions_handle) { auto new_opt = new ROCKSDB_NAMESPACE::ColumnFamilyOptions( *reinterpret_cast(joptions_handle)); @@ -3943,11 +3966,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptionsFromOptions( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: getColumnFamilyOptionsFromProps * Signature: (JLjava/lang/String;)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__JLjava_lang_String_2( +jlong Java_org_forstdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__JLjava_lang_String_2( JNIEnv* env, jclass, jlong cfg_handle, jstring jopt_string) { const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); if (opt_string == nullptr) { @@ -3977,11 +4000,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__JLja } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: getColumnFamilyOptionsFromProps * Signature: (Ljava/util/String;)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__Ljava_lang_String_2( +jlong Java_org_forstdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__Ljava_lang_String_2( JNIEnv* env, jclass, jstring jopt_string) { const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); if (opt_string == nullptr) { @@ -4013,11 +4036,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__Ljav } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_ColumnFamilyOptions_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_ColumnFamilyOptions_disposeInternal(JNIEnv*, jobject, jlong handle) { auto* cfo = reinterpret_cast(handle); assert(cfo != nullptr); @@ -4025,11 +4048,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: oldDefaults * Signature: (JII)V */ -void Java_org_rocksdb_ColumnFamilyOptions_oldDefaults(JNIEnv*, jclass, +void Java_org_forstdb_ColumnFamilyOptions_oldDefaults(JNIEnv*, jclass, jlong jhandle, jint major_version, jint minor_version) { @@ -4038,11 +4061,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_oldDefaults(JNIEnv*, jclass, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: optimizeForSmallDb * Signature: (J)V */ -void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb__J(JNIEnv*, +void Java_org_forstdb_ColumnFamilyOptions_optimizeForSmallDb__J(JNIEnv*, jobject, jlong jhandle) { reinterpret_cast(jhandle) @@ -4050,11 +4073,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb__J(JNIEnv*, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: optimizeForSmallDb * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb__JJ( +void Java_org_forstdb_ColumnFamilyOptions_optimizeForSmallDb__JJ( JNIEnv*, jclass, jlong jhandle, jlong cache_handle) { auto* cache_sptr_ptr = reinterpret_cast*>( @@ -4064,44 +4087,44 @@ void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb__JJ( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: optimizeForPointLookup * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_optimizeForPointLookup( +void Java_org_forstdb_ColumnFamilyOptions_optimizeForPointLookup( JNIEnv*, jobject, jlong jhandle, jlong block_cache_size_mb) { reinterpret_cast(jhandle) ->OptimizeForPointLookup(block_cache_size_mb); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: optimizeLevelStyleCompaction * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_optimizeLevelStyleCompaction( +void Java_org_forstdb_ColumnFamilyOptions_optimizeLevelStyleCompaction( JNIEnv*, jobject, jlong jhandle, jlong memtable_memory_budget) { reinterpret_cast(jhandle) ->OptimizeLevelStyleCompaction(memtable_memory_budget); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: optimizeUniversalStyleCompaction * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_optimizeUniversalStyleCompaction( +void Java_org_forstdb_ColumnFamilyOptions_optimizeUniversalStyleCompaction( JNIEnv*, jobject, jlong jhandle, jlong memtable_memory_budget) { reinterpret_cast(jhandle) ->OptimizeUniversalStyleCompaction(memtable_memory_budget); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setComparatorHandle * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setComparatorHandle__JI( +void Java_org_forstdb_ColumnFamilyOptions_setComparatorHandle__JI( JNIEnv*, jobject, jlong jhandle, jint builtinComparator) { switch (builtinComparator) { case 1: @@ -4116,11 +4139,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setComparatorHandle__JI( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setComparatorHandle * Signature: (JJB)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setComparatorHandle__JJB( +void Java_org_forstdb_ColumnFamilyOptions_setComparatorHandle__JJB( JNIEnv*, jobject, jlong jopt_handle, jlong jcomparator_handle, jbyte jcomparator_type) { ROCKSDB_NAMESPACE::Comparator* comparator = nullptr; @@ -4143,11 +4166,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setComparatorHandle__JJB( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMergeOperatorName * Signature: (JJjava/lang/String)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMergeOperatorName( +void Java_org_forstdb_ColumnFamilyOptions_setMergeOperatorName( JNIEnv* env, jobject, jlong jhandle, jstring jop_name) { auto* options = reinterpret_cast(jhandle); @@ -4163,11 +4186,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMergeOperatorName( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMergeOperator * Signature: (JJjava/lang/String)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMergeOperator( +void Java_org_forstdb_ColumnFamilyOptions_setMergeOperator( JNIEnv*, jobject, jlong jhandle, jlong mergeOperatorHandle) { reinterpret_cast(jhandle) ->merge_operator = @@ -4176,11 +4199,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMergeOperator( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompactionFilterHandle * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompactionFilterHandle( +void Java_org_forstdb_ColumnFamilyOptions_setCompactionFilterHandle( JNIEnv*, jobject, jlong jopt_handle, jlong jcompactionfilter_handle) { reinterpret_cast(jopt_handle) ->compaction_filter = @@ -4189,11 +4212,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompactionFilterHandle( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompactionFilterFactoryHandle * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompactionFilterFactoryHandle( +void Java_org_forstdb_ColumnFamilyOptions_setCompactionFilterFactoryHandle( JNIEnv*, jobject, jlong jopt_handle, jlong jcompactionfilterfactory_handle) { auto* cff_factory = reinterpret_cast< @@ -4204,11 +4227,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompactionFilterFactoryHandle( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setWriteBufferSize * Signature: (JJ)I */ -void Java_org_rocksdb_ColumnFamilyOptions_setWriteBufferSize( +void Java_org_forstdb_ColumnFamilyOptions_setWriteBufferSize( JNIEnv* env, jobject, jlong jhandle, jlong jwrite_buffer_size) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( jwrite_buffer_size); @@ -4221,33 +4244,33 @@ void Java_org_rocksdb_ColumnFamilyOptions_setWriteBufferSize( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: writeBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_writeBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_ColumnFamilyOptions_writeBufferSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->write_buffer_size; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxWriteBufferNumber * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxWriteBufferNumber( +void Java_org_forstdb_ColumnFamilyOptions_setMaxWriteBufferNumber( JNIEnv*, jobject, jlong jhandle, jint jmax_write_buffer_number) { reinterpret_cast(jhandle) ->max_write_buffer_number = jmax_write_buffer_number; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxWriteBufferNumber * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_maxWriteBufferNumber(JNIEnv*, jobject, +jint Java_org_forstdb_ColumnFamilyOptions_maxWriteBufferNumber(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_write_buffer_number; @@ -4257,7 +4280,7 @@ jint Java_org_rocksdb_ColumnFamilyOptions_maxWriteBufferNumber(JNIEnv*, jobject, * Method: setMemTableFactory * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMemTableFactory( +void Java_org_forstdb_ColumnFamilyOptions_setMemTableFactory( JNIEnv*, jobject, jlong jhandle, jlong jfactory_handle) { reinterpret_cast(jhandle) ->memtable_factory.reset( @@ -4266,11 +4289,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemTableFactory( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: memTableFactoryName * Signature: (J)Ljava/lang/String */ -jstring Java_org_rocksdb_ColumnFamilyOptions_memTableFactoryName( +jstring Java_org_forstdb_ColumnFamilyOptions_memTableFactoryName( JNIEnv* env, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -4292,7 +4315,7 @@ jstring Java_org_rocksdb_ColumnFamilyOptions_memTableFactoryName( * Method: useFixedLengthPrefixExtractor * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_useFixedLengthPrefixExtractor( +void Java_org_forstdb_ColumnFamilyOptions_useFixedLengthPrefixExtractor( JNIEnv*, jobject, jlong jhandle, jint jprefix_length) { reinterpret_cast(jhandle) ->prefix_extractor.reset(ROCKSDB_NAMESPACE::NewFixedPrefixTransform( @@ -4303,7 +4326,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_useFixedLengthPrefixExtractor( * Method: useCappedPrefixExtractor * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_useCappedPrefixExtractor( +void Java_org_forstdb_ColumnFamilyOptions_useCappedPrefixExtractor( JNIEnv*, jobject, jlong jhandle, jint jprefix_length) { reinterpret_cast(jhandle) ->prefix_extractor.reset(ROCKSDB_NAMESPACE::NewCappedPrefixTransform( @@ -4314,7 +4337,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_useCappedPrefixExtractor( * Method: setTableFactory * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setTableFactory( +void Java_org_forstdb_ColumnFamilyOptions_setTableFactory( JNIEnv*, jobject, jlong jhandle, jlong jfactory_handle) { reinterpret_cast(jhandle) ->table_factory.reset( @@ -4325,7 +4348,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_setTableFactory( * Method: setSstPartitionerFactory * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setSstPartitionerFactory( +void Java_org_forstdb_ColumnFamilyOptions_setSstPartitionerFactory( JNIEnv*, jobject, jlong jhandle, jlong factory_handle) { auto* options = reinterpret_cast(jhandle); @@ -4336,11 +4359,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setSstPartitionerFactory( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompactionThreadLimiter * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompactionThreadLimiter( +void Java_org_forstdb_ColumnFamilyOptions_setCompactionThreadLimiter( JNIEnv*, jclass, jlong jhandle, jlong jlimiter_handle) { auto* options = reinterpret_cast(jhandle); @@ -4354,7 +4377,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompactionThreadLimiter( * Method: tableFactoryName * Signature: (J)Ljava/lang/String */ -jstring Java_org_rocksdb_ColumnFamilyOptions_tableFactoryName(JNIEnv* env, +jstring Java_org_forstdb_ColumnFamilyOptions_tableFactoryName(JNIEnv* env, jobject, jlong jhandle) { auto* opt = @@ -4369,11 +4392,11 @@ jstring Java_org_rocksdb_ColumnFamilyOptions_tableFactoryName(JNIEnv* env, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCfPaths * Signature: (J[Ljava/lang/String;[J)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCfPaths(JNIEnv* env, jclass, +void Java_org_forstdb_ColumnFamilyOptions_setCfPaths(JNIEnv* env, jclass, jlong jhandle, jobjectArray path_array, jlongArray size_array) { @@ -4389,11 +4412,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCfPaths(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: cfPathsLen * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_cfPathsLen(JNIEnv*, jclass, +jlong Java_org_forstdb_ColumnFamilyOptions_cfPathsLen(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); @@ -4401,11 +4424,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_cfPathsLen(JNIEnv*, jclass, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: cfPaths * Signature: (J[Ljava/lang/String;[J)V */ -void Java_org_rocksdb_ColumnFamilyOptions_cfPaths(JNIEnv* env, jclass, +void Java_org_forstdb_ColumnFamilyOptions_cfPaths(JNIEnv* env, jclass, jlong jhandle, jobjectArray jpaths, jlongArray jtarget_sizes) { @@ -4415,22 +4438,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_cfPaths(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: minWriteBufferNumberToMerge * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_minWriteBufferNumberToMerge( +jint Java_org_forstdb_ColumnFamilyOptions_minWriteBufferNumberToMerge( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->min_write_buffer_number_to_merge; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMinWriteBufferNumberToMerge * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMinWriteBufferNumberToMerge( +void Java_org_forstdb_ColumnFamilyOptions_setMinWriteBufferNumberToMerge( JNIEnv*, jobject, jlong jhandle, jint jmin_write_buffer_number_to_merge) { reinterpret_cast(jhandle) ->min_write_buffer_number_to_merge = @@ -4438,22 +4461,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMinWriteBufferNumberToMerge( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxWriteBufferNumberToMaintain * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_maxWriteBufferNumberToMaintain( +jint Java_org_forstdb_ColumnFamilyOptions_maxWriteBufferNumberToMaintain( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_write_buffer_number_to_maintain; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxWriteBufferNumberToMaintain * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxWriteBufferNumberToMaintain( +void Java_org_forstdb_ColumnFamilyOptions_setMaxWriteBufferNumberToMaintain( JNIEnv*, jobject, jlong jhandle, jint jmax_write_buffer_number_to_maintain) { reinterpret_cast(jhandle) @@ -4462,11 +4485,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxWriteBufferNumberToMaintain( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompressionType * Signature: (JB)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompressionType( +void Java_org_forstdb_ColumnFamilyOptions_setCompressionType( JNIEnv*, jobject, jlong jhandle, jbyte jcompression_type_value) { auto* cf_opts = reinterpret_cast(jhandle); @@ -4476,11 +4499,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompressionType( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: compressionType * Signature: (J)B */ -jbyte Java_org_rocksdb_ColumnFamilyOptions_compressionType(JNIEnv*, jobject, +jbyte Java_org_forstdb_ColumnFamilyOptions_compressionType(JNIEnv*, jobject, jlong jhandle) { auto* cf_opts = reinterpret_cast(jhandle); @@ -4489,11 +4512,11 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_compressionType(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompressionPerLevel * Signature: (J[B)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel( +void Java_org_forstdb_ColumnFamilyOptions_setCompressionPerLevel( JNIEnv* env, jobject, jlong jhandle, jbyteArray jcompressionLevels) { auto* options = reinterpret_cast(jhandle); @@ -4507,11 +4530,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: compressionPerLevel * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_ColumnFamilyOptions_compressionPerLevel( +jbyteArray Java_org_forstdb_ColumnFamilyOptions_compressionPerLevel( JNIEnv* env, jobject, jlong jhandle) { auto* cf_options = reinterpret_cast(jhandle); @@ -4520,11 +4543,11 @@ jbyteArray Java_org_rocksdb_ColumnFamilyOptions_compressionPerLevel( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBottommostCompressionType * Signature: (JB)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBottommostCompressionType( +void Java_org_forstdb_ColumnFamilyOptions_setBottommostCompressionType( JNIEnv*, jobject, jlong jhandle, jbyte jcompression_type_value) { auto* cf_options = reinterpret_cast(jhandle); @@ -4534,11 +4557,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBottommostCompressionType( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: bottommostCompressionType * Signature: (J)B */ -jbyte Java_org_rocksdb_ColumnFamilyOptions_bottommostCompressionType( +jbyte Java_org_forstdb_ColumnFamilyOptions_bottommostCompressionType( JNIEnv*, jobject, jlong jhandle) { auto* cf_options = reinterpret_cast(jhandle); @@ -4546,11 +4569,11 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_bottommostCompressionType( cf_options->bottommost_compression); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBottommostCompressionOptions * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBottommostCompressionOptions( +void Java_org_forstdb_ColumnFamilyOptions_setBottommostCompressionOptions( JNIEnv*, jobject, jlong jhandle, jlong jbottommost_compression_options_handle) { auto* cf_options = @@ -4562,11 +4585,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBottommostCompressionOptions( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompressionOptions * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompressionOptions( +void Java_org_forstdb_ColumnFamilyOptions_setCompressionOptions( JNIEnv*, jobject, jlong jhandle, jlong jcompression_options_handle) { auto* cf_options = reinterpret_cast(jhandle); @@ -4577,11 +4600,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompressionOptions( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompactionStyle * Signature: (JB)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompactionStyle( +void Java_org_forstdb_ColumnFamilyOptions_setCompactionStyle( JNIEnv*, jobject, jlong jhandle, jbyte jcompaction_style) { auto* cf_options = reinterpret_cast(jhandle); @@ -4591,11 +4614,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompactionStyle( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: compactionStyle * Signature: (J)B */ -jbyte Java_org_rocksdb_ColumnFamilyOptions_compactionStyle(JNIEnv*, jobject, +jbyte Java_org_forstdb_ColumnFamilyOptions_compactionStyle(JNIEnv*, jobject, jlong jhandle) { auto* cf_options = reinterpret_cast(jhandle); @@ -4604,11 +4627,11 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_compactionStyle(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxTableFilesSizeFIFO * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxTableFilesSizeFIFO( +void Java_org_forstdb_ColumnFamilyOptions_setMaxTableFilesSizeFIFO( JNIEnv*, jobject, jlong jhandle, jlong jmax_table_files_size) { reinterpret_cast(jhandle) ->compaction_options_fifo.max_table_files_size = @@ -4616,33 +4639,33 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxTableFilesSizeFIFO( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxTableFilesSizeFIFO * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_maxTableFilesSizeFIFO( +jlong Java_org_forstdb_ColumnFamilyOptions_maxTableFilesSizeFIFO( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->compaction_options_fifo.max_table_files_size; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: numLevels * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_numLevels(JNIEnv*, jobject, +jint Java_org_forstdb_ColumnFamilyOptions_numLevels(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->num_levels; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setNumLevels * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setNumLevels(JNIEnv*, jobject, +void Java_org_forstdb_ColumnFamilyOptions_setNumLevels(JNIEnv*, jobject, jlong jhandle, jint jnum_levels) { reinterpret_cast(jhandle) @@ -4650,22 +4673,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setNumLevels(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: levelZeroFileNumCompactionTrigger * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_levelZeroFileNumCompactionTrigger( +jint Java_org_forstdb_ColumnFamilyOptions_levelZeroFileNumCompactionTrigger( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_file_num_compaction_trigger; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setLevelZeroFileNumCompactionTrigger * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroFileNumCompactionTrigger( +void Java_org_forstdb_ColumnFamilyOptions_setLevelZeroFileNumCompactionTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_file_num_compaction_trigger) { reinterpret_cast(jhandle) ->level0_file_num_compaction_trigger = @@ -4673,22 +4696,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroFileNumCompactionTrigger( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: levelZeroSlowdownWritesTrigger * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_levelZeroSlowdownWritesTrigger( +jint Java_org_forstdb_ColumnFamilyOptions_levelZeroSlowdownWritesTrigger( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_slowdown_writes_trigger; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setLevelSlowdownWritesTrigger * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroSlowdownWritesTrigger( +void Java_org_forstdb_ColumnFamilyOptions_setLevelZeroSlowdownWritesTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_slowdown_writes_trigger) { reinterpret_cast(jhandle) ->level0_slowdown_writes_trigger = @@ -4696,22 +4719,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroSlowdownWritesTrigger( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: levelZeroStopWritesTrigger * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_levelZeroStopWritesTrigger( +jint Java_org_forstdb_ColumnFamilyOptions_levelZeroStopWritesTrigger( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_stop_writes_trigger; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setLevelStopWritesTrigger * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroStopWritesTrigger( +void Java_org_forstdb_ColumnFamilyOptions_setLevelZeroStopWritesTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_stop_writes_trigger) { reinterpret_cast(jhandle) ->level0_stop_writes_trigger = @@ -4719,44 +4742,44 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroStopWritesTrigger( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: targetFileSizeBase * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_targetFileSizeBase(JNIEnv*, jobject, +jlong Java_org_forstdb_ColumnFamilyOptions_targetFileSizeBase(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->target_file_size_base; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setTargetFileSizeBase * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setTargetFileSizeBase( +void Java_org_forstdb_ColumnFamilyOptions_setTargetFileSizeBase( JNIEnv*, jobject, jlong jhandle, jlong jtarget_file_size_base) { reinterpret_cast(jhandle) ->target_file_size_base = static_cast(jtarget_file_size_base); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: targetFileSizeMultiplier * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_targetFileSizeMultiplier( +jint Java_org_forstdb_ColumnFamilyOptions_targetFileSizeMultiplier( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->target_file_size_multiplier; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setTargetFileSizeMultiplier * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setTargetFileSizeMultiplier( +void Java_org_forstdb_ColumnFamilyOptions_setTargetFileSizeMultiplier( JNIEnv*, jobject, jlong jhandle, jint jtarget_file_size_multiplier) { reinterpret_cast(jhandle) ->target_file_size_multiplier = @@ -4764,11 +4787,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setTargetFileSizeMultiplier( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxBytesForLevelBase * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelBase(JNIEnv*, +jlong Java_org_forstdb_ColumnFamilyOptions_maxBytesForLevelBase(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) @@ -4776,11 +4799,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelBase(JNIEnv*, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxBytesForLevelBase * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelBase( +void Java_org_forstdb_ColumnFamilyOptions_setMaxBytesForLevelBase( JNIEnv*, jobject, jlong jhandle, jlong jmax_bytes_for_level_base) { reinterpret_cast(jhandle) ->max_bytes_for_level_base = @@ -4788,44 +4811,44 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelBase( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: levelCompactionDynamicLevelBytes * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_levelCompactionDynamicLevelBytes( +jboolean Java_org_forstdb_ColumnFamilyOptions_levelCompactionDynamicLevelBytes( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level_compaction_dynamic_level_bytes; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setLevelCompactionDynamicLevelBytes * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setLevelCompactionDynamicLevelBytes( +void Java_org_forstdb_ColumnFamilyOptions_setLevelCompactionDynamicLevelBytes( JNIEnv*, jobject, jlong jhandle, jboolean jenable_dynamic_level_bytes) { reinterpret_cast(jhandle) ->level_compaction_dynamic_level_bytes = (jenable_dynamic_level_bytes); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxBytesForLevelMultiplier * Signature: (J)D */ -jdouble Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplier( +jdouble Java_org_forstdb_ColumnFamilyOptions_maxBytesForLevelMultiplier( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_bytes_for_level_multiplier; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxBytesForLevelMultiplier * Signature: (JD)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplier( +void Java_org_forstdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplier( JNIEnv*, jobject, jlong jhandle, jdouble jmax_bytes_for_level_multiplier) { reinterpret_cast(jhandle) ->max_bytes_for_level_multiplier = @@ -4833,11 +4856,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplier( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxCompactionBytes * Signature: (J)I */ -jlong Java_org_rocksdb_ColumnFamilyOptions_maxCompactionBytes(JNIEnv*, jobject, +jlong Java_org_forstdb_ColumnFamilyOptions_maxCompactionBytes(JNIEnv*, jobject, jlong jhandle) { return static_cast( reinterpret_cast(jhandle) @@ -4845,33 +4868,33 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_maxCompactionBytes(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxCompactionBytes * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxCompactionBytes( +void Java_org_forstdb_ColumnFamilyOptions_setMaxCompactionBytes( JNIEnv*, jobject, jlong jhandle, jlong jmax_compaction_bytes) { reinterpret_cast(jhandle) ->max_compaction_bytes = static_cast(jmax_compaction_bytes); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: arenaBlockSize * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_arenaBlockSize(JNIEnv*, jobject, +jlong Java_org_forstdb_ColumnFamilyOptions_arenaBlockSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->arena_block_size; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setArenaBlockSize * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setArenaBlockSize( +void Java_org_forstdb_ColumnFamilyOptions_setArenaBlockSize( JNIEnv* env, jobject, jlong jhandle, jlong jarena_block_size) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jarena_block_size); @@ -4884,44 +4907,44 @@ void Java_org_rocksdb_ColumnFamilyOptions_setArenaBlockSize( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: disableAutoCompactions * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_disableAutoCompactions( +jboolean Java_org_forstdb_ColumnFamilyOptions_disableAutoCompactions( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->disable_auto_compactions; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setDisableAutoCompactions * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setDisableAutoCompactions( +void Java_org_forstdb_ColumnFamilyOptions_setDisableAutoCompactions( JNIEnv*, jobject, jlong jhandle, jboolean jdisable_auto_compactions) { reinterpret_cast(jhandle) ->disable_auto_compactions = static_cast(jdisable_auto_compactions); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxSequentialSkipInIterations * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_maxSequentialSkipInIterations( +jlong Java_org_forstdb_ColumnFamilyOptions_maxSequentialSkipInIterations( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_sequential_skip_in_iterations; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxSequentialSkipInIterations * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxSequentialSkipInIterations( +void Java_org_forstdb_ColumnFamilyOptions_setMaxSequentialSkipInIterations( JNIEnv*, jobject, jlong jhandle, jlong jmax_sequential_skip_in_iterations) { reinterpret_cast(jhandle) ->max_sequential_skip_in_iterations = @@ -4929,44 +4952,44 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxSequentialSkipInIterations( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: inplaceUpdateSupport * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_inplaceUpdateSupport( +jboolean Java_org_forstdb_ColumnFamilyOptions_inplaceUpdateSupport( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->inplace_update_support; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setInplaceUpdateSupport * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateSupport( +void Java_org_forstdb_ColumnFamilyOptions_setInplaceUpdateSupport( JNIEnv*, jobject, jlong jhandle, jboolean jinplace_update_support) { reinterpret_cast(jhandle) ->inplace_update_support = static_cast(jinplace_update_support); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: inplaceUpdateNumLocks * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_inplaceUpdateNumLocks( +jlong Java_org_forstdb_ColumnFamilyOptions_inplaceUpdateNumLocks( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->inplace_update_num_locks; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setInplaceUpdateNumLocks * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateNumLocks( +void Java_org_forstdb_ColumnFamilyOptions_setInplaceUpdateNumLocks( JNIEnv* env, jobject, jlong jhandle, jlong jinplace_update_num_locks) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( jinplace_update_num_locks); @@ -4979,22 +5002,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateNumLocks( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: memtablePrefixBloomSizeRatio * Signature: (J)I */ -jdouble Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomSizeRatio( +jdouble Java_org_forstdb_ColumnFamilyOptions_memtablePrefixBloomSizeRatio( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->memtable_prefix_bloom_size_ratio; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMemtablePrefixBloomSizeRatio * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio( +void Java_org_forstdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio( JNIEnv*, jobject, jlong jhandle, jdouble jmemtable_prefix_bloom_size_ratio) { reinterpret_cast(jhandle) @@ -5003,22 +5026,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: experimentalMempurgeThreshold * Signature: (J)I */ -jdouble Java_org_rocksdb_ColumnFamilyOptions_experimentalMempurgeThreshold( +jdouble Java_org_forstdb_ColumnFamilyOptions_experimentalMempurgeThreshold( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->experimental_mempurge_threshold; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setExperimentalMempurgeThreshold * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setExperimentalMempurgeThreshold( +void Java_org_forstdb_ColumnFamilyOptions_setExperimentalMempurgeThreshold( JNIEnv*, jobject, jlong jhandle, jdouble jexperimental_mempurge_threshold) { reinterpret_cast(jhandle) ->experimental_mempurge_threshold = @@ -5026,22 +5049,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setExperimentalMempurgeThreshold( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: memtableWholeKeyFiltering * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_memtableWholeKeyFiltering( +jboolean Java_org_forstdb_ColumnFamilyOptions_memtableWholeKeyFiltering( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->memtable_whole_key_filtering; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMemtableWholeKeyFiltering * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering( +void Java_org_forstdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering( JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) { reinterpret_cast(jhandle) ->memtable_whole_key_filtering = @@ -5049,44 +5072,44 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: bloomLocality * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_bloomLocality(JNIEnv*, jobject, +jint Java_org_forstdb_ColumnFamilyOptions_bloomLocality(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->bloom_locality; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBloomLocality * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBloomLocality( +void Java_org_forstdb_ColumnFamilyOptions_setBloomLocality( JNIEnv*, jobject, jlong jhandle, jint jbloom_locality) { reinterpret_cast(jhandle) ->bloom_locality = static_cast(jbloom_locality); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxSuccessiveMerges * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_maxSuccessiveMerges(JNIEnv*, jobject, +jlong Java_org_forstdb_ColumnFamilyOptions_maxSuccessiveMerges(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_successive_merges; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxSuccessiveMerges * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxSuccessiveMerges( +void Java_org_forstdb_ColumnFamilyOptions_setMaxSuccessiveMerges( JNIEnv* env, jobject, jlong jhandle, jlong jmax_successive_merges) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( jmax_successive_merges); @@ -5099,22 +5122,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxSuccessiveMerges( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: optimizeFiltersForHits * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_optimizeFiltersForHits( +jboolean Java_org_forstdb_ColumnFamilyOptions_optimizeFiltersForHits( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->optimize_filters_for_hits; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setOptimizeFiltersForHits * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setOptimizeFiltersForHits( +void Java_org_forstdb_ColumnFamilyOptions_setOptimizeFiltersForHits( JNIEnv*, jobject, jlong jhandle, jboolean joptimize_filters_for_hits) { reinterpret_cast(jhandle) ->optimize_filters_for_hits = @@ -5122,11 +5145,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setOptimizeFiltersForHits( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: memtableHugePageSize * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_memtableHugePageSize(JNIEnv*, +jlong Java_org_forstdb_ColumnFamilyOptions_memtableHugePageSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) @@ -5134,11 +5157,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_memtableHugePageSize(JNIEnv*, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMemtableHugePageSize * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMemtableHugePageSize( +void Java_org_forstdb_ColumnFamilyOptions_setMemtableHugePageSize( JNIEnv* env, jobject, jlong jhandle, jlong jmemtable_huge_page_size) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( jmemtable_huge_page_size); @@ -5151,22 +5174,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemtableHugePageSize( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: softPendingCompactionBytesLimit * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_softPendingCompactionBytesLimit( +jlong Java_org_forstdb_ColumnFamilyOptions_softPendingCompactionBytesLimit( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->soft_pending_compaction_bytes_limit; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setSoftPendingCompactionBytesLimit * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setSoftPendingCompactionBytesLimit( +void Java_org_forstdb_ColumnFamilyOptions_setSoftPendingCompactionBytesLimit( JNIEnv*, jobject, jlong jhandle, jlong jsoft_pending_compaction_bytes_limit) { reinterpret_cast(jhandle) @@ -5175,22 +5198,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setSoftPendingCompactionBytesLimit( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: softHardCompactionBytesLimit * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_hardPendingCompactionBytesLimit( +jlong Java_org_forstdb_ColumnFamilyOptions_hardPendingCompactionBytesLimit( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->hard_pending_compaction_bytes_limit; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setHardPendingCompactionBytesLimit * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setHardPendingCompactionBytesLimit( +void Java_org_forstdb_ColumnFamilyOptions_setHardPendingCompactionBytesLimit( JNIEnv*, jobject, jlong jhandle, jlong jhard_pending_compaction_bytes_limit) { reinterpret_cast(jhandle) @@ -5199,22 +5222,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setHardPendingCompactionBytesLimit( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: level0FileNumCompactionTrigger * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_level0FileNumCompactionTrigger( +jint Java_org_forstdb_ColumnFamilyOptions_level0FileNumCompactionTrigger( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_file_num_compaction_trigger; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setLevel0FileNumCompactionTrigger * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setLevel0FileNumCompactionTrigger( +void Java_org_forstdb_ColumnFamilyOptions_setLevel0FileNumCompactionTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_file_num_compaction_trigger) { reinterpret_cast(jhandle) ->level0_file_num_compaction_trigger = @@ -5222,22 +5245,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevel0FileNumCompactionTrigger( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: level0SlowdownWritesTrigger * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_level0SlowdownWritesTrigger( +jint Java_org_forstdb_ColumnFamilyOptions_level0SlowdownWritesTrigger( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_slowdown_writes_trigger; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setLevel0SlowdownWritesTrigger * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setLevel0SlowdownWritesTrigger( +void Java_org_forstdb_ColumnFamilyOptions_setLevel0SlowdownWritesTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_slowdown_writes_trigger) { reinterpret_cast(jhandle) ->level0_slowdown_writes_trigger = @@ -5245,22 +5268,22 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevel0SlowdownWritesTrigger( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: level0StopWritesTrigger * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_level0StopWritesTrigger( +jint Java_org_forstdb_ColumnFamilyOptions_level0StopWritesTrigger( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->level0_stop_writes_trigger; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setLevel0StopWritesTrigger * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setLevel0StopWritesTrigger( +void Java_org_forstdb_ColumnFamilyOptions_setLevel0StopWritesTrigger( JNIEnv*, jobject, jlong jhandle, jint jlevel0_stop_writes_trigger) { reinterpret_cast(jhandle) ->level0_stop_writes_trigger = @@ -5268,12 +5291,12 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevel0StopWritesTrigger( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: maxBytesForLevelMultiplierAdditional * Signature: (J)[I */ jintArray -Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplierAdditional( +Java_org_forstdb_ColumnFamilyOptions_maxBytesForLevelMultiplierAdditional( JNIEnv* env, jobject, jlong jhandle) { auto mbflma = reinterpret_cast(jhandle) @@ -5307,11 +5330,11 @@ Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplierAdditional( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMaxBytesForLevelMultiplierAdditional * Signature: (J[I)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplierAdditional( +void Java_org_forstdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplierAdditional( JNIEnv* env, jobject, jlong jhandle, jintArray jmax_bytes_for_level_multiplier_additional) { jsize len = env->GetArrayLength(jmax_bytes_for_level_multiplier_additional); @@ -5335,33 +5358,33 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplierAdditiona } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: paranoidFileChecks * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_paranoidFileChecks( +jboolean Java_org_forstdb_ColumnFamilyOptions_paranoidFileChecks( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->paranoid_file_checks; } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setParanoidFileChecks * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setParanoidFileChecks( +void Java_org_forstdb_ColumnFamilyOptions_setParanoidFileChecks( JNIEnv*, jobject, jlong jhandle, jboolean jparanoid_file_checks) { reinterpret_cast(jhandle) ->paranoid_file_checks = static_cast(jparanoid_file_checks); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompactionPriority * Signature: (JB)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompactionPriority( +void Java_org_forstdb_ColumnFamilyOptions_setCompactionPriority( JNIEnv*, jobject, jlong jhandle, jbyte jcompaction_priority_value) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5371,11 +5394,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompactionPriority( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: compactionPriority * Signature: (J)B */ -jbyte Java_org_rocksdb_ColumnFamilyOptions_compactionPriority(JNIEnv*, jobject, +jbyte Java_org_forstdb_ColumnFamilyOptions_compactionPriority(JNIEnv*, jobject, jlong jhandle) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5384,11 +5407,11 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_compactionPriority(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setReportBgIoStats * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setReportBgIoStats( +void Java_org_forstdb_ColumnFamilyOptions_setReportBgIoStats( JNIEnv*, jobject, jlong jhandle, jboolean jreport_bg_io_stats) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5396,11 +5419,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setReportBgIoStats( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: reportBgIoStats * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_reportBgIoStats(JNIEnv*, jobject, +jboolean Java_org_forstdb_ColumnFamilyOptions_reportBgIoStats(JNIEnv*, jobject, jlong jhandle) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5408,11 +5431,11 @@ jboolean Java_org_rocksdb_ColumnFamilyOptions_reportBgIoStats(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setTtl * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setTtl(JNIEnv*, jobject, +void Java_org_forstdb_ColumnFamilyOptions_setTtl(JNIEnv*, jobject, jlong jhandle, jlong jttl) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5420,23 +5443,23 @@ void Java_org_rocksdb_ColumnFamilyOptions_setTtl(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: ttl * Signature: (J)J */ JNIEXPORT jlong JNICALL -Java_org_rocksdb_ColumnFamilyOptions_ttl(JNIEnv*, jobject, jlong jhandle) { +Java_org_forstdb_ColumnFamilyOptions_ttl(JNIEnv*, jobject, jlong jhandle) { auto* cf_opts = reinterpret_cast(jhandle); return static_cast(cf_opts->ttl); } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setPeriodicCompactionSeconds * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setPeriodicCompactionSeconds( +void Java_org_forstdb_ColumnFamilyOptions_setPeriodicCompactionSeconds( JNIEnv*, jobject, jlong jhandle, jlong jperiodicCompactionSeconds) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5445,12 +5468,12 @@ void Java_org_rocksdb_ColumnFamilyOptions_setPeriodicCompactionSeconds( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: periodicCompactionSeconds * Signature: (J)J */ JNIEXPORT jlong JNICALL -Java_org_rocksdb_ColumnFamilyOptions_periodicCompactionSeconds(JNIEnv*, jobject, +Java_org_forstdb_ColumnFamilyOptions_periodicCompactionSeconds(JNIEnv*, jobject, jlong jhandle) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5458,11 +5481,11 @@ Java_org_rocksdb_ColumnFamilyOptions_periodicCompactionSeconds(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompactionOptionsUniversal * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompactionOptionsUniversal( +void Java_org_forstdb_ColumnFamilyOptions_setCompactionOptionsUniversal( JNIEnv*, jobject, jlong jhandle, jlong jcompaction_options_universal_handle) { auto* cf_opts = @@ -5474,11 +5497,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompactionOptionsUniversal( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setCompactionOptionsFIFO * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setCompactionOptionsFIFO( +void Java_org_forstdb_ColumnFamilyOptions_setCompactionOptionsFIFO( JNIEnv*, jobject, jlong jhandle, jlong jcompaction_options_fifo_handle) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5488,11 +5511,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompactionOptionsFIFO( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setForceConsistencyChecks * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setForceConsistencyChecks( +void Java_org_forstdb_ColumnFamilyOptions_setForceConsistencyChecks( JNIEnv*, jobject, jlong jhandle, jboolean jforce_consistency_checks) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5501,11 +5524,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setForceConsistencyChecks( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: forceConsistencyChecks * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_forceConsistencyChecks( +jboolean Java_org_forstdb_ColumnFamilyOptions_forceConsistencyChecks( JNIEnv*, jobject, jlong jhandle) { auto* cf_opts = reinterpret_cast(jhandle); @@ -5515,11 +5538,11 @@ jboolean Java_org_rocksdb_ColumnFamilyOptions_forceConsistencyChecks( /// BLOB options /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setEnableBlobFiles * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setEnableBlobFiles( +void Java_org_forstdb_ColumnFamilyOptions_setEnableBlobFiles( JNIEnv*, jobject, jlong jhandle, jboolean jenable_blob_files) { auto* opts = reinterpret_cast(jhandle); @@ -5527,11 +5550,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setEnableBlobFiles( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: enableBlobFiles * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_enableBlobFiles(JNIEnv*, jobject, +jboolean Java_org_forstdb_ColumnFamilyOptions_enableBlobFiles(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -5539,11 +5562,11 @@ jboolean Java_org_rocksdb_ColumnFamilyOptions_enableBlobFiles(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setMinBlobSize * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setMinBlobSize(JNIEnv*, jobject, +void Java_org_forstdb_ColumnFamilyOptions_setMinBlobSize(JNIEnv*, jobject, jlong jhandle, jlong jmin_blob_size) { auto* opts = @@ -5552,11 +5575,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMinBlobSize(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: minBlobSize * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_minBlobSize(JNIEnv*, jobject, +jlong Java_org_forstdb_ColumnFamilyOptions_minBlobSize(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -5564,11 +5587,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_minBlobSize(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBlobFileSize * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBlobFileSize( +void Java_org_forstdb_ColumnFamilyOptions_setBlobFileSize( JNIEnv*, jobject, jlong jhandle, jlong jblob_file_size) { auto* opts = reinterpret_cast(jhandle); @@ -5576,11 +5599,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBlobFileSize( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: blobFileSize * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_blobFileSize(JNIEnv*, jobject, +jlong Java_org_forstdb_ColumnFamilyOptions_blobFileSize(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -5588,11 +5611,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_blobFileSize(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBlobCompressionType * Signature: (JB)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBlobCompressionType( +void Java_org_forstdb_ColumnFamilyOptions_setBlobCompressionType( JNIEnv*, jobject, jlong jhandle, jbyte jblob_compression_type_value) { auto* opts = reinterpret_cast(jhandle); @@ -5602,11 +5625,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBlobCompressionType( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: blobCompressionType * Signature: (J)B */ -jbyte Java_org_rocksdb_ColumnFamilyOptions_blobCompressionType(JNIEnv*, jobject, +jbyte Java_org_forstdb_ColumnFamilyOptions_blobCompressionType(JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -5615,11 +5638,11 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_blobCompressionType(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setEnableBlobGarbageCollection * Signature: (JZ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setEnableBlobGarbageCollection( +void Java_org_forstdb_ColumnFamilyOptions_setEnableBlobGarbageCollection( JNIEnv*, jobject, jlong jhandle, jboolean jenable_blob_garbage_collection) { auto* opts = reinterpret_cast(jhandle); @@ -5628,11 +5651,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setEnableBlobGarbageCollection( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: enableBlobGarbageCollection * Signature: (J)Z */ -jboolean Java_org_rocksdb_ColumnFamilyOptions_enableBlobGarbageCollection( +jboolean Java_org_forstdb_ColumnFamilyOptions_enableBlobGarbageCollection( JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -5640,11 +5663,11 @@ jboolean Java_org_rocksdb_ColumnFamilyOptions_enableBlobGarbageCollection( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBlobGarbageCollectionAgeCutoff * Signature: (JD)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionAgeCutoff( +void Java_org_forstdb_ColumnFamilyOptions_setBlobGarbageCollectionAgeCutoff( JNIEnv*, jobject, jlong jhandle, jdouble jblob_garbage_collection_age_cutoff) { auto* opts = @@ -5654,11 +5677,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionAgeCutoff( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: blobGarbageCollectionAgeCutoff * Signature: (J)D */ -jdouble Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionAgeCutoff( +jdouble Java_org_forstdb_ColumnFamilyOptions_blobGarbageCollectionAgeCutoff( JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -5666,11 +5689,11 @@ jdouble Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionAgeCutoff( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBlobGarbageCollectionForceThreshold * Signature: (JD)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold( +void Java_org_forstdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold( JNIEnv*, jobject, jlong jhandle, jdouble jblob_garbage_collection_force_threshold) { auto* opts = @@ -5680,12 +5703,12 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: blobGarbageCollectionForceThreshold * Signature: (J)D */ jdouble -Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold( +Java_org_forstdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold( JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -5693,11 +5716,11 @@ Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBlobCompactionReadaheadSize * Signature: (JJ)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBlobCompactionReadaheadSize( +void Java_org_forstdb_ColumnFamilyOptions_setBlobCompactionReadaheadSize( JNIEnv*, jobject, jlong jhandle, jlong jblob_compaction_readahead_size) { auto* opts = reinterpret_cast(jhandle); @@ -5706,11 +5729,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBlobCompactionReadaheadSize( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: blobCompactionReadaheadSize * Signature: (J)J */ -jlong Java_org_rocksdb_ColumnFamilyOptions_blobCompactionReadaheadSize( +jlong Java_org_forstdb_ColumnFamilyOptions_blobCompactionReadaheadSize( JNIEnv*, jobject, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -5718,11 +5741,11 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_blobCompactionReadaheadSize( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setBlobFileStartingLevel * Signature: (JI)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setBlobFileStartingLevel( +void Java_org_forstdb_ColumnFamilyOptions_setBlobFileStartingLevel( JNIEnv*, jobject, jlong jhandle, jint jblob_file_starting_level) { auto* opts = reinterpret_cast(jhandle); @@ -5730,11 +5753,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBlobFileStartingLevel( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: blobFileStartingLevel * Signature: (J)I */ -jint Java_org_rocksdb_ColumnFamilyOptions_blobFileStartingLevel(JNIEnv*, +jint Java_org_forstdb_ColumnFamilyOptions_blobFileStartingLevel(JNIEnv*, jobject, jlong jhandle) { auto* opts = @@ -5743,11 +5766,11 @@ jint Java_org_rocksdb_ColumnFamilyOptions_blobFileStartingLevel(JNIEnv*, } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: setPrepopulateBlobCache * Signature: (JB)V */ -void Java_org_rocksdb_ColumnFamilyOptions_setPrepopulateBlobCache( +void Java_org_forstdb_ColumnFamilyOptions_setPrepopulateBlobCache( JNIEnv*, jobject, jlong jhandle, jbyte jprepopulate_blob_cache_value) { auto* opts = reinterpret_cast(jhandle); @@ -5757,11 +5780,11 @@ void Java_org_rocksdb_ColumnFamilyOptions_setPrepopulateBlobCache( } /* - * Class: org_rocksdb_ColumnFamilyOptions + * Class: org_forstdb_ColumnFamilyOptions * Method: prepopulateBlobCache * Signature: (J)B */ -jbyte Java_org_rocksdb_ColumnFamilyOptions_prepopulateBlobCache(JNIEnv*, +jbyte Java_org_forstdb_ColumnFamilyOptions_prepopulateBlobCache(JNIEnv*, jobject, jlong jhandle) { auto* opts = @@ -5770,36 +5793,60 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_prepopulateBlobCache(JNIEnv*, opts->prepopulate_blob_cache); } +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMemtableMaxRangeDeletions + * Signature: (JI)V + */ +void Java_org_forstdb_ColumnFamilyOptions_setMemtableMaxRangeDeletions( + JNIEnv*, jobject, jlong jhandle, jint jmemtable_max_range_deletions) { + auto* opts = + reinterpret_cast(jhandle); + opts->memtable_max_range_deletions = jmemtable_max_range_deletions; +} + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: memtableMaxRangeDeletions + * Signature: (J)I + */ +jint Java_org_forstdb_ColumnFamilyOptions_memtableMaxRangeDeletions( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->memtable_max_range_deletions); +} + ///////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::DBOptions /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: newDBOptions * Signature: ()J */ -jlong Java_org_rocksdb_DBOptions_newDBOptions(JNIEnv*, jclass) { +jlong Java_org_forstdb_DBOptions_newDBOptions(JNIEnv*, jclass) { auto* dbop = new ROCKSDB_NAMESPACE::DBOptions(); return GET_CPLUSPLUS_POINTER(dbop); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: copyDBOptions * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_copyDBOptions(JNIEnv*, jclass, jlong jhandle) { +jlong Java_org_forstdb_DBOptions_copyDBOptions(JNIEnv*, jclass, jlong jhandle) { auto new_opt = new ROCKSDB_NAMESPACE::DBOptions( *(reinterpret_cast(jhandle))); return GET_CPLUSPLUS_POINTER(new_opt); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: newDBOptionsFromOptions * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_newDBOptionsFromOptions( +jlong Java_org_forstdb_DBOptions_newDBOptionsFromOptions( JNIEnv*, jclass, jlong joptions_handle) { auto new_opt = new ROCKSDB_NAMESPACE::DBOptions( *reinterpret_cast(joptions_handle)); @@ -5807,11 +5854,11 @@ jlong Java_org_rocksdb_DBOptions_newDBOptionsFromOptions( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: getDBOptionsFromProps * Signature: (JLjava/lang/String;)J */ -jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps__JLjava_lang_String_2( +jlong Java_org_forstdb_DBOptions_getDBOptionsFromProps__JLjava_lang_String_2( JNIEnv* env, jclass, jlong config_handle, jstring jopt_string) { const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); if (opt_string == nullptr) { @@ -5840,11 +5887,11 @@ jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps__JLjava_lang_String_2( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: getDBOptionsFromProps * Signature: (Ljava/util/String;)J */ -jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps__Ljava_lang_String_2( +jlong Java_org_forstdb_DBOptions_getDBOptionsFromProps__Ljava_lang_String_2( JNIEnv* env, jclass, jstring jopt_string) { const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); if (opt_string == nullptr) { @@ -5875,11 +5922,11 @@ jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps__Ljava_lang_String_2( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_DBOptions_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_disposeInternal(JNIEnv*, jobject, jlong handle) { auto* dbo = reinterpret_cast(handle); assert(dbo != nullptr); @@ -5887,33 +5934,33 @@ void Java_org_rocksdb_DBOptions_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: optimizeForSmallDb * Signature: (J)V */ -void Java_org_rocksdb_DBOptions_optimizeForSmallDb(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_optimizeForSmallDb(JNIEnv*, jobject, jlong jhandle) { reinterpret_cast(jhandle) ->OptimizeForSmallDb(); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setEnv * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setEnv(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setEnv(JNIEnv*, jobject, jlong jhandle, jlong jenv_handle) { reinterpret_cast(jhandle)->env = reinterpret_cast(jenv_handle); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setIncreaseParallelism * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setIncreaseParallelism(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setIncreaseParallelism(JNIEnv*, jobject, jlong jhandle, jint totalThreads) { reinterpret_cast(jhandle)->IncreaseParallelism( @@ -5921,11 +5968,11 @@ void Java_org_rocksdb_DBOptions_setIncreaseParallelism(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setCreateIfMissing * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setCreateIfMissing(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setCreateIfMissing(JNIEnv*, jobject, jlong jhandle, jboolean flag) { reinterpret_cast(jhandle)->create_if_missing = @@ -5933,22 +5980,22 @@ void Java_org_rocksdb_DBOptions_setCreateIfMissing(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: createIfMissing * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_createIfMissing(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_createIfMissing(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->create_if_missing; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setCreateMissingColumnFamilies * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setCreateMissingColumnFamilies(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setCreateMissingColumnFamilies(JNIEnv*, jobject, jlong jhandle, jboolean flag) { reinterpret_cast(jhandle) @@ -5956,11 +6003,11 @@ void Java_org_rocksdb_DBOptions_setCreateMissingColumnFamilies(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: createMissingColumnFamilies * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_createMissingColumnFamilies(JNIEnv*, +jboolean Java_org_forstdb_DBOptions_createMissingColumnFamilies(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) @@ -5968,11 +6015,11 @@ jboolean Java_org_rocksdb_DBOptions_createMissingColumnFamilies(JNIEnv*, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setErrorIfExists * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setErrorIfExists(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setErrorIfExists(JNIEnv*, jobject, jlong jhandle, jboolean error_if_exists) { reinterpret_cast(jhandle)->error_if_exists = @@ -5980,22 +6027,22 @@ void Java_org_rocksdb_DBOptions_setErrorIfExists(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: errorIfExists * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_errorIfExists(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_errorIfExists(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->error_if_exists; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setParanoidChecks * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setParanoidChecks(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setParanoidChecks(JNIEnv*, jobject, jlong jhandle, jboolean paranoid_checks) { reinterpret_cast(jhandle)->paranoid_checks = @@ -6003,22 +6050,22 @@ void Java_org_rocksdb_DBOptions_setParanoidChecks(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: paranoidChecks * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_paranoidChecks(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_paranoidChecks(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->paranoid_checks; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setRateLimiter * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setRateLimiter(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setRateLimiter(JNIEnv*, jobject, jlong jhandle, jlong jrate_limiter_handle) { std::shared_ptr* pRateLimiter = reinterpret_cast*>( @@ -6028,11 +6075,11 @@ void Java_org_rocksdb_DBOptions_setRateLimiter(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setSstFileManager * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setSstFileManager( +void Java_org_forstdb_DBOptions_setSstFileManager( JNIEnv*, jobject, jlong jhandle, jlong jsst_file_manager_handle) { auto* sptr_sst_file_manager = reinterpret_cast*>( @@ -6042,11 +6089,11 @@ void Java_org_rocksdb_DBOptions_setSstFileManager( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setLogger * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setLogger(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setLogger(JNIEnv*, jobject, jlong jhandle, jlong jlogger_handle) { std::shared_ptr* pLogger = reinterpret_cast*>( @@ -6055,32 +6102,32 @@ void Java_org_rocksdb_DBOptions_setLogger(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setInfoLogLevel * Signature: (JB)V */ -void Java_org_rocksdb_DBOptions_setInfoLogLevel(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setInfoLogLevel(JNIEnv*, jobject, jlong jhandle, jbyte jlog_level) { reinterpret_cast(jhandle)->info_log_level = static_cast(jlog_level); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: infoLogLevel * Signature: (J)B */ -jbyte Java_org_rocksdb_DBOptions_infoLogLevel(JNIEnv*, jobject, jlong jhandle) { +jbyte Java_org_forstdb_DBOptions_infoLogLevel(JNIEnv*, jobject, jlong jhandle) { return static_cast( reinterpret_cast(jhandle)->info_log_level); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxTotalWalSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setMaxTotalWalSize(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setMaxTotalWalSize(JNIEnv*, jobject, jlong jhandle, jlong jmax_total_wal_size) { reinterpret_cast(jhandle)->max_total_wal_size = @@ -6088,65 +6135,65 @@ void Java_org_rocksdb_DBOptions_setMaxTotalWalSize(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxTotalWalSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_maxTotalWalSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_maxTotalWalSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_total_wal_size; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxOpenFiles * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setMaxOpenFiles(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setMaxOpenFiles(JNIEnv*, jobject, jlong jhandle, jint max_open_files) { reinterpret_cast(jhandle)->max_open_files = static_cast(max_open_files); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxOpenFiles * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_maxOpenFiles(JNIEnv*, jobject, jlong jhandle) { +jint Java_org_forstdb_DBOptions_maxOpenFiles(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_open_files; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxFileOpeningThreads * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setMaxFileOpeningThreads( +void Java_org_forstdb_DBOptions_setMaxFileOpeningThreads( JNIEnv*, jobject, jlong jhandle, jint jmax_file_opening_threads) { reinterpret_cast(jhandle) ->max_file_opening_threads = static_cast(jmax_file_opening_threads); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxFileOpeningThreads * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_maxFileOpeningThreads(JNIEnv*, jobject, +jint Java_org_forstdb_DBOptions_maxFileOpeningThreads(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_file_opening_threads); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setStatistics * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setStatistics(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setStatistics(JNIEnv*, jobject, jlong jhandle, jlong jstatistics_handle) { auto* opt = reinterpret_cast(jhandle); auto* pSptr = @@ -6156,11 +6203,11 @@ void Java_org_rocksdb_DBOptions_setStatistics(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: statistics * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_statistics(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_DBOptions_statistics(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); std::shared_ptr sptr = opt->statistics; if (sptr == nullptr) { @@ -6173,31 +6220,31 @@ jlong Java_org_rocksdb_DBOptions_statistics(JNIEnv*, jobject, jlong jhandle) { } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setUseFsync * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setUseFsync(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setUseFsync(JNIEnv*, jobject, jlong jhandle, jboolean use_fsync) { reinterpret_cast(jhandle)->use_fsync = static_cast(use_fsync); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: useFsync * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_useFsync(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_DBOptions_useFsync(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->use_fsync; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setDbPaths * Signature: (J[Ljava/lang/String;[J)V */ -void Java_org_rocksdb_DBOptions_setDbPaths(JNIEnv* env, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setDbPaths(JNIEnv* env, jobject, jlong jhandle, jobjectArray jpaths, jlongArray jtarget_sizes) { std::vector db_paths; @@ -6239,21 +6286,21 @@ void Java_org_rocksdb_DBOptions_setDbPaths(JNIEnv* env, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: dbPathsLen * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_dbPathsLen(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_DBOptions_dbPathsLen(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->db_paths.size()); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: dbPaths * Signature: (J[Ljava/lang/String;[J)V */ -void Java_org_rocksdb_DBOptions_dbPaths(JNIEnv* env, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_dbPaths(JNIEnv* env, jobject, jlong jhandle, jobjectArray jpaths, jlongArray jtarget_sizes) { jboolean is_copy; @@ -6290,11 +6337,11 @@ void Java_org_rocksdb_DBOptions_dbPaths(JNIEnv* env, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setDbLogDir * Signature: (JLjava/lang/String)V */ -void Java_org_rocksdb_DBOptions_setDbLogDir(JNIEnv* env, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setDbLogDir(JNIEnv* env, jobject, jlong jhandle, jstring jdb_log_dir) { const char* log_dir = env->GetStringUTFChars(jdb_log_dir, nullptr); if (log_dir == nullptr) { @@ -6308,11 +6355,11 @@ void Java_org_rocksdb_DBOptions_setDbLogDir(JNIEnv* env, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: dbLogDir * Signature: (J)Ljava/lang/String */ -jstring Java_org_rocksdb_DBOptions_dbLogDir(JNIEnv* env, jobject, +jstring Java_org_forstdb_DBOptions_dbLogDir(JNIEnv* env, jobject, jlong jhandle) { return env->NewStringUTF( reinterpret_cast(jhandle) @@ -6320,11 +6367,11 @@ jstring Java_org_rocksdb_DBOptions_dbLogDir(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWalDir * Signature: (JLjava/lang/String)V */ -void Java_org_rocksdb_DBOptions_setWalDir(JNIEnv* env, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setWalDir(JNIEnv* env, jobject, jlong jhandle, jstring jwal_dir) { const char* wal_dir = env->GetStringUTFChars(jwal_dir, 0); reinterpret_cast(jhandle)->wal_dir.assign( @@ -6333,44 +6380,44 @@ void Java_org_rocksdb_DBOptions_setWalDir(JNIEnv* env, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: walDir * Signature: (J)Ljava/lang/String */ -jstring Java_org_rocksdb_DBOptions_walDir(JNIEnv* env, jobject, jlong jhandle) { +jstring Java_org_forstdb_DBOptions_walDir(JNIEnv* env, jobject, jlong jhandle) { return env->NewStringUTF( reinterpret_cast(jhandle) ->wal_dir.c_str()); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setDeleteObsoleteFilesPeriodMicros * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setDeleteObsoleteFilesPeriodMicros( +void Java_org_forstdb_DBOptions_setDeleteObsoleteFilesPeriodMicros( JNIEnv*, jobject, jlong jhandle, jlong micros) { reinterpret_cast(jhandle) ->delete_obsolete_files_period_micros = static_cast(micros); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: deleteObsoleteFilesPeriodMicros * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_deleteObsoleteFilesPeriodMicros( +jlong Java_org_forstdb_DBOptions_deleteObsoleteFilesPeriodMicros( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->delete_obsolete_files_period_micros; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxBackgroundCompactions * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setMaxBackgroundCompactions(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setMaxBackgroundCompactions(JNIEnv*, jobject, jlong jhandle, jint max) { reinterpret_cast(jhandle) @@ -6378,66 +6425,66 @@ void Java_org_rocksdb_DBOptions_setMaxBackgroundCompactions(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxBackgroundCompactions * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_maxBackgroundCompactions(JNIEnv*, jobject, +jint Java_org_forstdb_DBOptions_maxBackgroundCompactions(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_background_compactions; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxSubcompactions * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setMaxSubcompactions(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setMaxSubcompactions(JNIEnv*, jobject, jlong jhandle, jint max) { reinterpret_cast(jhandle)->max_subcompactions = static_cast(max); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxSubcompactions * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_maxSubcompactions(JNIEnv*, jobject, +jint Java_org_forstdb_DBOptions_maxSubcompactions(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_subcompactions; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxBackgroundFlushes * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setMaxBackgroundFlushes( +void Java_org_forstdb_DBOptions_setMaxBackgroundFlushes( JNIEnv*, jobject, jlong jhandle, jint max_background_flushes) { reinterpret_cast(jhandle) ->max_background_flushes = static_cast(max_background_flushes); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxBackgroundFlushes * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_maxBackgroundFlushes(JNIEnv*, jobject, +jint Java_org_forstdb_DBOptions_maxBackgroundFlushes(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_background_flushes; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxBackgroundJobs * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setMaxBackgroundJobs(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setMaxBackgroundJobs(JNIEnv*, jobject, jlong jhandle, jint max_background_jobs) { reinterpret_cast(jhandle) @@ -6445,22 +6492,22 @@ void Java_org_rocksdb_DBOptions_setMaxBackgroundJobs(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxBackgroundJobs * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_maxBackgroundJobs(JNIEnv*, jobject, +jint Java_org_forstdb_DBOptions_maxBackgroundJobs(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_background_jobs; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxLogFileSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setMaxLogFileSize(JNIEnv* env, jobject, +void Java_org_forstdb_DBOptions_setMaxLogFileSize(JNIEnv* env, jobject, jlong jhandle, jlong max_log_file_size) { auto s = @@ -6474,22 +6521,22 @@ void Java_org_rocksdb_DBOptions_setMaxLogFileSize(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxLogFileSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_maxLogFileSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_maxLogFileSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_log_file_size; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setLogFileTimeToRoll * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setLogFileTimeToRoll( +void Java_org_forstdb_DBOptions_setLogFileTimeToRoll( JNIEnv* env, jobject, jlong jhandle, jlong log_file_time_to_roll) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( log_file_time_to_roll); @@ -6502,22 +6549,22 @@ void Java_org_rocksdb_DBOptions_setLogFileTimeToRoll( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: logFileTimeToRoll * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_logFileTimeToRoll(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_logFileTimeToRoll(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->log_file_time_to_roll; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setKeepLogFileNum * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setKeepLogFileNum(JNIEnv* env, jobject, +void Java_org_forstdb_DBOptions_setKeepLogFileNum(JNIEnv* env, jobject, jlong jhandle, jlong keep_log_file_num) { auto s = @@ -6531,22 +6578,22 @@ void Java_org_rocksdb_DBOptions_setKeepLogFileNum(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: keepLogFileNum * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_keepLogFileNum(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_keepLogFileNum(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->keep_log_file_num; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setRecycleLogFileNum * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setRecycleLogFileNum( +void Java_org_forstdb_DBOptions_setRecycleLogFileNum( JNIEnv* env, jobject, jlong jhandle, jlong recycle_log_file_num) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( recycle_log_file_num); @@ -6559,66 +6606,66 @@ void Java_org_rocksdb_DBOptions_setRecycleLogFileNum( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: recycleLogFileNum * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_recycleLogFileNum(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_recycleLogFileNum(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->recycle_log_file_num; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxManifestFileSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setMaxManifestFileSize( +void Java_org_forstdb_DBOptions_setMaxManifestFileSize( JNIEnv*, jobject, jlong jhandle, jlong max_manifest_file_size) { reinterpret_cast(jhandle) ->max_manifest_file_size = static_cast(max_manifest_file_size); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxManifestFileSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_maxManifestFileSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_maxManifestFileSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->max_manifest_file_size; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setTableCacheNumshardbits * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setTableCacheNumshardbits( +void Java_org_forstdb_DBOptions_setTableCacheNumshardbits( JNIEnv*, jobject, jlong jhandle, jint table_cache_numshardbits) { reinterpret_cast(jhandle) ->table_cache_numshardbits = static_cast(table_cache_numshardbits); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: tableCacheNumshardbits * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_tableCacheNumshardbits(JNIEnv*, jobject, +jint Java_org_forstdb_DBOptions_tableCacheNumshardbits(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->table_cache_numshardbits; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWalTtlSeconds * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setWalTtlSeconds(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setWalTtlSeconds(JNIEnv*, jobject, jlong jhandle, jlong WAL_ttl_seconds) { reinterpret_cast(jhandle)->WAL_ttl_seconds = @@ -6626,22 +6673,22 @@ void Java_org_rocksdb_DBOptions_setWalTtlSeconds(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: walTtlSeconds * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_walTtlSeconds(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_walTtlSeconds(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->WAL_ttl_seconds; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWalSizeLimitMB * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setWalSizeLimitMB(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setWalSizeLimitMB(JNIEnv*, jobject, jlong jhandle, jlong WAL_size_limit_MB) { reinterpret_cast(jhandle)->WAL_size_limit_MB = @@ -6649,22 +6696,22 @@ void Java_org_rocksdb_DBOptions_setWalSizeLimitMB(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: walTtlSeconds * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_walSizeLimitMB(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_walSizeLimitMB(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->WAL_size_limit_MB; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxWriteBatchGroupSizeBytes * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setMaxWriteBatchGroupSizeBytes( +void Java_org_forstdb_DBOptions_setMaxWriteBatchGroupSizeBytes( JNIEnv*, jclass, jlong jhandle, jlong jmax_write_batch_group_size_bytes) { auto* opt = reinterpret_cast(jhandle); opt->max_write_batch_group_size_bytes = @@ -6672,22 +6719,22 @@ void Java_org_rocksdb_DBOptions_setMaxWriteBatchGroupSizeBytes( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxWriteBatchGroupSizeBytes * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_maxWriteBatchGroupSizeBytes(JNIEnv*, jclass, +jlong Java_org_forstdb_DBOptions_maxWriteBatchGroupSizeBytes(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_write_batch_group_size_bytes); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setManifestPreallocationSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setManifestPreallocationSize( +void Java_org_forstdb_DBOptions_setManifestPreallocationSize( JNIEnv* env, jobject, jlong jhandle, jlong preallocation_size) { auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( preallocation_size); @@ -6700,33 +6747,33 @@ void Java_org_rocksdb_DBOptions_setManifestPreallocationSize( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: manifestPreallocationSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_manifestPreallocationSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_manifestPreallocationSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->manifest_preallocation_size; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: useDirectReads * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_useDirectReads(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_useDirectReads(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->use_direct_reads; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setUseDirectReads * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setUseDirectReads(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setUseDirectReads(JNIEnv*, jobject, jlong jhandle, jboolean use_direct_reads) { reinterpret_cast(jhandle)->use_direct_reads = @@ -6734,22 +6781,22 @@ void Java_org_rocksdb_DBOptions_setUseDirectReads(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: useDirectIoForFlushAndCompaction * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_useDirectIoForFlushAndCompaction( +jboolean Java_org_forstdb_DBOptions_useDirectIoForFlushAndCompaction( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->use_direct_io_for_flush_and_compaction; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setUseDirectReads * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setUseDirectIoForFlushAndCompaction( +void Java_org_forstdb_DBOptions_setUseDirectIoForFlushAndCompaction( JNIEnv*, jobject, jlong jhandle, jboolean use_direct_io_for_flush_and_compaction) { reinterpret_cast(jhandle) @@ -6758,11 +6805,11 @@ void Java_org_rocksdb_DBOptions_setUseDirectIoForFlushAndCompaction( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAllowFAllocate * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAllowFAllocate(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setAllowFAllocate(JNIEnv*, jobject, jlong jhandle, jboolean jallow_fallocate) { reinterpret_cast(jhandle)->allow_fallocate = @@ -6770,22 +6817,22 @@ void Java_org_rocksdb_DBOptions_setAllowFAllocate(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: allowFAllocate * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_allowFAllocate(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_allowFAllocate(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->allow_fallocate); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAllowMmapReads * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAllowMmapReads(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setAllowMmapReads(JNIEnv*, jobject, jlong jhandle, jboolean allow_mmap_reads) { reinterpret_cast(jhandle)->allow_mmap_reads = @@ -6793,22 +6840,22 @@ void Java_org_rocksdb_DBOptions_setAllowMmapReads(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: allowMmapReads * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_allowMmapReads(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_allowMmapReads(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->allow_mmap_reads; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAllowMmapWrites * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAllowMmapWrites(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setAllowMmapWrites(JNIEnv*, jobject, jlong jhandle, jboolean allow_mmap_writes) { reinterpret_cast(jhandle)->allow_mmap_writes = @@ -6816,44 +6863,44 @@ void Java_org_rocksdb_DBOptions_setAllowMmapWrites(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: allowMmapWrites * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_allowMmapWrites(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_allowMmapWrites(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->allow_mmap_writes; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setIsFdCloseOnExec * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setIsFdCloseOnExec( +void Java_org_forstdb_DBOptions_setIsFdCloseOnExec( JNIEnv*, jobject, jlong jhandle, jboolean is_fd_close_on_exec) { reinterpret_cast(jhandle) ->is_fd_close_on_exec = static_cast(is_fd_close_on_exec); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: isFdCloseOnExec * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_isFdCloseOnExec(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_isFdCloseOnExec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->is_fd_close_on_exec; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setStatsDumpPeriodSec * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setStatsDumpPeriodSec( +void Java_org_forstdb_DBOptions_setStatsDumpPeriodSec( JNIEnv*, jobject, jlong jhandle, jint jstats_dump_period_sec) { reinterpret_cast(jhandle) ->stats_dump_period_sec = @@ -6861,22 +6908,22 @@ void Java_org_rocksdb_DBOptions_setStatsDumpPeriodSec( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: statsDumpPeriodSec * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_statsDumpPeriodSec(JNIEnv*, jobject, +jint Java_org_forstdb_DBOptions_statsDumpPeriodSec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->stats_dump_period_sec; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setStatsPersistPeriodSec * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setStatsPersistPeriodSec( +void Java_org_forstdb_DBOptions_setStatsPersistPeriodSec( JNIEnv*, jobject, jlong jhandle, jint jstats_persist_period_sec) { reinterpret_cast(jhandle) ->stats_persist_period_sec = @@ -6884,22 +6931,22 @@ void Java_org_rocksdb_DBOptions_setStatsPersistPeriodSec( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: statsPersistPeriodSec * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_statsPersistPeriodSec(JNIEnv*, jobject, +jint Java_org_forstdb_DBOptions_statsPersistPeriodSec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->stats_persist_period_sec; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setStatsHistoryBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setStatsHistoryBufferSize( +void Java_org_forstdb_DBOptions_setStatsHistoryBufferSize( JNIEnv*, jobject, jlong jhandle, jlong jstats_history_buffer_size) { reinterpret_cast(jhandle) ->stats_history_buffer_size = @@ -6907,55 +6954,55 @@ void Java_org_rocksdb_DBOptions_setStatsHistoryBufferSize( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: statsHistoryBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_statsHistoryBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_statsHistoryBufferSize(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->stats_history_buffer_size; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAdviseRandomOnOpen * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAdviseRandomOnOpen( +void Java_org_forstdb_DBOptions_setAdviseRandomOnOpen( JNIEnv*, jobject, jlong jhandle, jboolean advise_random_on_open) { reinterpret_cast(jhandle) ->advise_random_on_open = static_cast(advise_random_on_open); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: adviseRandomOnOpen * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_adviseRandomOnOpen(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_adviseRandomOnOpen(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->advise_random_on_open; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setDbWriteBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setDbWriteBufferSize( +void Java_org_forstdb_DBOptions_setDbWriteBufferSize( JNIEnv*, jobject, jlong jhandle, jlong jdb_write_buffer_size) { auto* opt = reinterpret_cast(jhandle); opt->db_write_buffer_size = static_cast(jdb_write_buffer_size); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWriteBufferManager * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setWriteBufferManager( +void Java_org_forstdb_DBOptions_setWriteBufferManager( JNIEnv*, jobject, jlong jdb_options_handle, jlong jwrite_buffer_manager_handle) { auto* write_buffer_manager = @@ -6966,22 +7013,22 @@ void Java_org_rocksdb_DBOptions_setWriteBufferManager( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: dbWriteBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_dbWriteBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_dbWriteBufferSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->db_write_buffer_size); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAccessHintOnCompactionStart * Signature: (JB)V */ -void Java_org_rocksdb_DBOptions_setAccessHintOnCompactionStart( +void Java_org_forstdb_DBOptions_setAccessHintOnCompactionStart( JNIEnv*, jobject, jlong jhandle, jbyte jaccess_hint_value) { auto* opt = reinterpret_cast(jhandle); opt->access_hint_on_compaction_start = @@ -6989,11 +7036,11 @@ void Java_org_rocksdb_DBOptions_setAccessHintOnCompactionStart( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: accessHintOnCompactionStart * Signature: (J)B */ -jbyte Java_org_rocksdb_DBOptions_accessHintOnCompactionStart(JNIEnv*, jobject, +jbyte Java_org_forstdb_DBOptions_accessHintOnCompactionStart(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::AccessHintJni::toJavaAccessHint( @@ -7001,11 +7048,11 @@ jbyte Java_org_rocksdb_DBOptions_accessHintOnCompactionStart(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setCompactionReadaheadSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setCompactionReadaheadSize( +void Java_org_forstdb_DBOptions_setCompactionReadaheadSize( JNIEnv*, jobject, jlong jhandle, jlong jcompaction_readahead_size) { auto* opt = reinterpret_cast(jhandle); opt->compaction_readahead_size = @@ -7013,22 +7060,22 @@ void Java_org_rocksdb_DBOptions_setCompactionReadaheadSize( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: compactionReadaheadSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_compactionReadaheadSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_compactionReadaheadSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->compaction_readahead_size); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setRandomAccessMaxBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setRandomAccessMaxBufferSize( +void Java_org_forstdb_DBOptions_setRandomAccessMaxBufferSize( JNIEnv*, jobject, jlong jhandle, jlong jrandom_access_max_buffer_size) { auto* opt = reinterpret_cast(jhandle); opt->random_access_max_buffer_size = @@ -7036,22 +7083,22 @@ void Java_org_rocksdb_DBOptions_setRandomAccessMaxBufferSize( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: randomAccessMaxBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_randomAccessMaxBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_randomAccessMaxBufferSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->random_access_max_buffer_size); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWritableFileMaxBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setWritableFileMaxBufferSize( +void Java_org_forstdb_DBOptions_setWritableFileMaxBufferSize( JNIEnv*, jobject, jlong jhandle, jlong jwritable_file_max_buffer_size) { auto* opt = reinterpret_cast(jhandle); opt->writable_file_max_buffer_size = @@ -7059,65 +7106,65 @@ void Java_org_rocksdb_DBOptions_setWritableFileMaxBufferSize( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: writableFileMaxBufferSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_writableFileMaxBufferSize(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_writableFileMaxBufferSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->writable_file_max_buffer_size); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setUseAdaptiveMutex * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setUseAdaptiveMutex( +void Java_org_forstdb_DBOptions_setUseAdaptiveMutex( JNIEnv*, jobject, jlong jhandle, jboolean use_adaptive_mutex) { reinterpret_cast(jhandle)->use_adaptive_mutex = static_cast(use_adaptive_mutex); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: useAdaptiveMutex * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_useAdaptiveMutex(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_useAdaptiveMutex(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->use_adaptive_mutex; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setBytesPerSync * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setBytesPerSync(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setBytesPerSync(JNIEnv*, jobject, jlong jhandle, jlong bytes_per_sync) { reinterpret_cast(jhandle)->bytes_per_sync = static_cast(bytes_per_sync); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: bytesPerSync * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_bytesPerSync(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_DBOptions_bytesPerSync(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->bytes_per_sync; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWalBytesPerSync * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setWalBytesPerSync(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setWalBytesPerSync(JNIEnv*, jobject, jlong jhandle, jlong jwal_bytes_per_sync) { reinterpret_cast(jhandle)->wal_bytes_per_sync = @@ -7125,33 +7172,33 @@ void Java_org_rocksdb_DBOptions_setWalBytesPerSync(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: walBytesPerSync * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_walBytesPerSync(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_walBytesPerSync(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->wal_bytes_per_sync); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setStrictBytesPerSync * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setStrictBytesPerSync( +void Java_org_forstdb_DBOptions_setStrictBytesPerSync( JNIEnv*, jobject, jlong jhandle, jboolean jstrict_bytes_per_sync) { reinterpret_cast(jhandle) ->strict_bytes_per_sync = jstrict_bytes_per_sync == JNI_TRUE; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: strictBytesPerSync * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_strictBytesPerSync(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_strictBytesPerSync(JNIEnv*, jobject, jlong jhandle) { return static_cast( reinterpret_cast(jhandle) @@ -7159,11 +7206,11 @@ jboolean Java_org_rocksdb_DBOptions_strictBytesPerSync(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setEventListeners * Signature: (J[J)V */ -void Java_org_rocksdb_DBOptions_setEventListeners(JNIEnv* env, jclass, +void Java_org_forstdb_DBOptions_setEventListeners(JNIEnv* env, jclass, jlong jhandle, jlongArray jlistener_array) { auto* opt = reinterpret_cast(jhandle); @@ -7171,22 +7218,22 @@ void Java_org_rocksdb_DBOptions_setEventListeners(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: eventListeners - * Signature: (J)[Lorg/rocksdb/AbstractEventListener; + * Signature: (J)[Lorg/forstdb/AbstractEventListener; */ -jobjectArray Java_org_rocksdb_DBOptions_eventListeners(JNIEnv* env, jclass, +jobjectArray Java_org_forstdb_DBOptions_eventListeners(JNIEnv* env, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return rocksdb_get_event_listeners_helper(env, opt->listeners); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setDelayedWriteRate * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setDelayedWriteRate(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setDelayedWriteRate(JNIEnv*, jobject, jlong jhandle, jlong jdelayed_write_rate) { auto* opt = reinterpret_cast(jhandle); @@ -7194,44 +7241,44 @@ void Java_org_rocksdb_DBOptions_setDelayedWriteRate(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: delayedWriteRate * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_delayedWriteRate(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_delayedWriteRate(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->delayed_write_rate); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setEnablePipelinedWrite * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setEnablePipelinedWrite( +void Java_org_forstdb_DBOptions_setEnablePipelinedWrite( JNIEnv*, jobject, jlong jhandle, jboolean jenable_pipelined_write) { auto* opt = reinterpret_cast(jhandle); opt->enable_pipelined_write = jenable_pipelined_write == JNI_TRUE; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: enablePipelinedWrite * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_enablePipelinedWrite(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_enablePipelinedWrite(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->enable_pipelined_write); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setUnorderedWrite * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setUnorderedWrite(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setUnorderedWrite(JNIEnv*, jobject, jlong jhandle, jboolean junordered_write) { auto* opt = reinterpret_cast(jhandle); @@ -7239,88 +7286,88 @@ void Java_org_rocksdb_DBOptions_setUnorderedWrite(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: unorderedWrite * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_unorderedWrite(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_unorderedWrite(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->unordered_write); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setEnableThreadTracking * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setEnableThreadTracking( +void Java_org_forstdb_DBOptions_setEnableThreadTracking( JNIEnv*, jobject, jlong jhandle, jboolean jenable_thread_tracking) { auto* opt = reinterpret_cast(jhandle); opt->enable_thread_tracking = jenable_thread_tracking == JNI_TRUE; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: enableThreadTracking * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_enableThreadTracking(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_enableThreadTracking(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->enable_thread_tracking); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAllowConcurrentMemtableWrite * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAllowConcurrentMemtableWrite( +void Java_org_forstdb_DBOptions_setAllowConcurrentMemtableWrite( JNIEnv*, jobject, jlong jhandle, jboolean allow) { reinterpret_cast(jhandle) ->allow_concurrent_memtable_write = static_cast(allow); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: allowConcurrentMemtableWrite * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_allowConcurrentMemtableWrite( +jboolean Java_org_forstdb_DBOptions_allowConcurrentMemtableWrite( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->allow_concurrent_memtable_write; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setEnableWriteThreadAdaptiveYield * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setEnableWriteThreadAdaptiveYield( +void Java_org_forstdb_DBOptions_setEnableWriteThreadAdaptiveYield( JNIEnv*, jobject, jlong jhandle, jboolean yield) { reinterpret_cast(jhandle) ->enable_write_thread_adaptive_yield = static_cast(yield); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: enableWriteThreadAdaptiveYield * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_enableWriteThreadAdaptiveYield( +jboolean Java_org_forstdb_DBOptions_enableWriteThreadAdaptiveYield( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->enable_write_thread_adaptive_yield; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWriteThreadMaxYieldUsec * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setWriteThreadMaxYieldUsec(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setWriteThreadMaxYieldUsec(JNIEnv*, jobject, jlong jhandle, jlong max) { reinterpret_cast(jhandle) @@ -7328,22 +7375,22 @@ void Java_org_rocksdb_DBOptions_setWriteThreadMaxYieldUsec(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: writeThreadMaxYieldUsec * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_writeThreadMaxYieldUsec(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_writeThreadMaxYieldUsec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->write_thread_max_yield_usec; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWriteThreadSlowYieldUsec * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setWriteThreadSlowYieldUsec(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setWriteThreadSlowYieldUsec(JNIEnv*, jobject, jlong jhandle, jlong slow) { reinterpret_cast(jhandle) @@ -7351,22 +7398,22 @@ void Java_org_rocksdb_DBOptions_setWriteThreadSlowYieldUsec(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: writeThreadSlowYieldUsec * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_writeThreadSlowYieldUsec(JNIEnv*, jobject, +jlong Java_org_forstdb_DBOptions_writeThreadSlowYieldUsec(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->write_thread_slow_yield_usec; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setSkipStatsUpdateOnDbOpen * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setSkipStatsUpdateOnDbOpen( +void Java_org_forstdb_DBOptions_setSkipStatsUpdateOnDbOpen( JNIEnv*, jobject, jlong jhandle, jboolean jskip_stats_update_on_db_open) { auto* opt = reinterpret_cast(jhandle); opt->skip_stats_update_on_db_open = @@ -7374,22 +7421,22 @@ void Java_org_rocksdb_DBOptions_setSkipStatsUpdateOnDbOpen( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: skipStatsUpdateOnDbOpen * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_skipStatsUpdateOnDbOpen(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_skipStatsUpdateOnDbOpen(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->skip_stats_update_on_db_open); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setSkipCheckingSstFileSizesOnDbOpen * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setSkipCheckingSstFileSizesOnDbOpen( +void Java_org_forstdb_DBOptions_setSkipCheckingSstFileSizesOnDbOpen( JNIEnv*, jclass, jlong jhandle, jboolean jskip_checking_sst_file_sizes_on_db_open) { auto* opt = reinterpret_cast(jhandle); @@ -7398,22 +7445,22 @@ void Java_org_rocksdb_DBOptions_setSkipCheckingSstFileSizesOnDbOpen( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: skipCheckingSstFileSizesOnDbOpen * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_skipCheckingSstFileSizesOnDbOpen( +jboolean Java_org_forstdb_DBOptions_skipCheckingSstFileSizesOnDbOpen( JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->skip_checking_sst_file_sizes_on_db_open); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWalRecoveryMode * Signature: (JB)V */ -void Java_org_rocksdb_DBOptions_setWalRecoveryMode( +void Java_org_forstdb_DBOptions_setWalRecoveryMode( JNIEnv*, jobject, jlong jhandle, jbyte jwal_recovery_mode_value) { auto* opt = reinterpret_cast(jhandle); opt->wal_recovery_mode = @@ -7422,11 +7469,11 @@ void Java_org_rocksdb_DBOptions_setWalRecoveryMode( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: walRecoveryMode * Signature: (J)B */ -jbyte Java_org_rocksdb_DBOptions_walRecoveryMode(JNIEnv*, jobject, +jbyte Java_org_forstdb_DBOptions_walRecoveryMode(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return ROCKSDB_NAMESPACE::WALRecoveryModeJni::toJavaWALRecoveryMode( @@ -7434,32 +7481,32 @@ jbyte Java_org_rocksdb_DBOptions_walRecoveryMode(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAllow2pc * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAllow2pc(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setAllow2pc(JNIEnv*, jobject, jlong jhandle, jboolean jallow_2pc) { auto* opt = reinterpret_cast(jhandle); opt->allow_2pc = static_cast(jallow_2pc); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: allow2pc * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_allow2pc(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_DBOptions_allow2pc(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->allow_2pc); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setRowCache * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setRowCache(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setRowCache(JNIEnv*, jobject, jlong jhandle, jlong jrow_cache_handle) { auto* opt = reinterpret_cast(jhandle); auto* row_cache = @@ -7469,11 +7516,11 @@ void Java_org_rocksdb_DBOptions_setRowCache(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWalFilter * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setWalFilter(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setWalFilter(JNIEnv*, jobject, jlong jhandle, jlong jwal_filter_handle) { auto* opt = reinterpret_cast(jhandle); auto* wal_filter = reinterpret_cast( @@ -7482,11 +7529,11 @@ void Java_org_rocksdb_DBOptions_setWalFilter(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setFailIfOptionsFileError * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setFailIfOptionsFileError( +void Java_org_forstdb_DBOptions_setFailIfOptionsFileError( JNIEnv*, jobject, jlong jhandle, jboolean jfail_if_options_file_error) { auto* opt = reinterpret_cast(jhandle); opt->fail_if_options_file_error = @@ -7494,44 +7541,44 @@ void Java_org_rocksdb_DBOptions_setFailIfOptionsFileError( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: failIfOptionsFileError * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_failIfOptionsFileError(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_failIfOptionsFileError(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->fail_if_options_file_error); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setDumpMallocStats * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setDumpMallocStats( +void Java_org_forstdb_DBOptions_setDumpMallocStats( JNIEnv*, jobject, jlong jhandle, jboolean jdump_malloc_stats) { auto* opt = reinterpret_cast(jhandle); opt->dump_malloc_stats = static_cast(jdump_malloc_stats); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: dumpMallocStats * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_dumpMallocStats(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_dumpMallocStats(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->dump_malloc_stats); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAvoidFlushDuringRecovery * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAvoidFlushDuringRecovery( +void Java_org_forstdb_DBOptions_setAvoidFlushDuringRecovery( JNIEnv*, jobject, jlong jhandle, jboolean javoid_flush_during_recovery) { auto* opt = reinterpret_cast(jhandle); opt->avoid_flush_during_recovery = @@ -7539,44 +7586,44 @@ void Java_org_rocksdb_DBOptions_setAvoidFlushDuringRecovery( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: avoidFlushDuringRecovery * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_avoidFlushDuringRecovery(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_avoidFlushDuringRecovery(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->avoid_flush_during_recovery); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAllowIngestBehind * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAllowIngestBehind( +void Java_org_forstdb_DBOptions_setAllowIngestBehind( JNIEnv*, jobject, jlong jhandle, jboolean jallow_ingest_behind) { auto* opt = reinterpret_cast(jhandle); opt->allow_ingest_behind = jallow_ingest_behind == JNI_TRUE; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: allowIngestBehind * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_allowIngestBehind(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_allowIngestBehind(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->allow_ingest_behind); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setTwoWriteQueues * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setTwoWriteQueues(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setTwoWriteQueues(JNIEnv*, jobject, jlong jhandle, jboolean jtwo_write_queues) { auto* opt = reinterpret_cast(jhandle); @@ -7584,22 +7631,22 @@ void Java_org_rocksdb_DBOptions_setTwoWriteQueues(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: twoWriteQueues * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_twoWriteQueues(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_twoWriteQueues(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->two_write_queues); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setManualWalFlush * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setManualWalFlush(JNIEnv*, jobject, +void Java_org_forstdb_DBOptions_setManualWalFlush(JNIEnv*, jobject, jlong jhandle, jboolean jmanual_wal_flush) { auto* opt = reinterpret_cast(jhandle); @@ -7607,44 +7654,44 @@ void Java_org_rocksdb_DBOptions_setManualWalFlush(JNIEnv*, jobject, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: manualWalFlush * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_manualWalFlush(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_manualWalFlush(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->manual_wal_flush); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAtomicFlush * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAtomicFlush(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_DBOptions_setAtomicFlush(JNIEnv*, jobject, jlong jhandle, jboolean jatomic_flush) { auto* opt = reinterpret_cast(jhandle); opt->atomic_flush = jatomic_flush == JNI_TRUE; } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: atomicFlush * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_atomicFlush(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_atomicFlush(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->atomic_flush); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAvoidFlushDuringShutdown * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAvoidFlushDuringShutdown( +void Java_org_forstdb_DBOptions_setAvoidFlushDuringShutdown( JNIEnv*, jobject, jlong jhandle, jboolean javoid_flush_during_shutdown) { auto* opt = reinterpret_cast(jhandle); opt->avoid_flush_during_shutdown = @@ -7652,88 +7699,88 @@ void Java_org_rocksdb_DBOptions_setAvoidFlushDuringShutdown( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: avoidFlushDuringShutdown * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_avoidFlushDuringShutdown(JNIEnv*, jobject, +jboolean Java_org_forstdb_DBOptions_avoidFlushDuringShutdown(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->avoid_flush_during_shutdown); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setAvoidUnnecessaryBlockingIO * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setAvoidUnnecessaryBlockingIO( +void Java_org_forstdb_DBOptions_setAvoidUnnecessaryBlockingIO( JNIEnv*, jclass, jlong jhandle, jboolean avoid_blocking_io) { auto* opt = reinterpret_cast(jhandle); opt->avoid_unnecessary_blocking_io = static_cast(avoid_blocking_io); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: avoidUnnecessaryBlockingIO * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_avoidUnnecessaryBlockingIO(JNIEnv*, jclass, +jboolean Java_org_forstdb_DBOptions_avoidUnnecessaryBlockingIO(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->avoid_unnecessary_blocking_io); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setPersistStatsToDisk * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setPersistStatsToDisk( +void Java_org_forstdb_DBOptions_setPersistStatsToDisk( JNIEnv*, jclass, jlong jhandle, jboolean persist_stats_to_disk) { auto* opt = reinterpret_cast(jhandle); opt->persist_stats_to_disk = static_cast(persist_stats_to_disk); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: persistStatsToDisk * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_persistStatsToDisk(JNIEnv*, jclass, +jboolean Java_org_forstdb_DBOptions_persistStatsToDisk(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->persist_stats_to_disk); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setWriteDbidToManifest * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setWriteDbidToManifest( +void Java_org_forstdb_DBOptions_setWriteDbidToManifest( JNIEnv*, jclass, jlong jhandle, jboolean jwrite_dbid_to_manifest) { auto* opt = reinterpret_cast(jhandle); opt->write_dbid_to_manifest = static_cast(jwrite_dbid_to_manifest); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: writeDbidToManifest * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_writeDbidToManifest(JNIEnv*, jclass, +jboolean Java_org_forstdb_DBOptions_writeDbidToManifest(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->write_dbid_to_manifest); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setLogReadaheadSize * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setLogReadaheadSize(JNIEnv*, jclass, +void Java_org_forstdb_DBOptions_setLogReadaheadSize(JNIEnv*, jclass, jlong jhandle, jlong jlog_readahead_size) { auto* opt = reinterpret_cast(jhandle); @@ -7741,66 +7788,66 @@ void Java_org_rocksdb_DBOptions_setLogReadaheadSize(JNIEnv*, jclass, } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: logReasaheadSize * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_logReadaheadSize(JNIEnv*, jclass, +jlong Java_org_forstdb_DBOptions_logReadaheadSize(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->log_readahead_size); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setBestEffortsRecovery * Signature: (JZ)V */ -void Java_org_rocksdb_DBOptions_setBestEffortsRecovery( +void Java_org_forstdb_DBOptions_setBestEffortsRecovery( JNIEnv*, jclass, jlong jhandle, jboolean jbest_efforts_recovery) { auto* opt = reinterpret_cast(jhandle); opt->best_efforts_recovery = static_cast(jbest_efforts_recovery); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: bestEffortsRecovery * Signature: (J)Z */ -jboolean Java_org_rocksdb_DBOptions_bestEffortsRecovery(JNIEnv*, jclass, +jboolean Java_org_forstdb_DBOptions_bestEffortsRecovery(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->best_efforts_recovery); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setMaxBgErrorResumeCount * Signature: (JI)V */ -void Java_org_rocksdb_DBOptions_setMaxBgErrorResumeCount( +void Java_org_forstdb_DBOptions_setMaxBgErrorResumeCount( JNIEnv*, jclass, jlong jhandle, jint jmax_bgerror_resume_count) { auto* opt = reinterpret_cast(jhandle); opt->max_bgerror_resume_count = static_cast(jmax_bgerror_resume_count); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: maxBgerrorResumeCount * Signature: (J)I */ -jint Java_org_rocksdb_DBOptions_maxBgerrorResumeCount(JNIEnv*, jclass, +jint Java_org_forstdb_DBOptions_maxBgerrorResumeCount(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_bgerror_resume_count); } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: setBgerrorResumeRetryInterval * Signature: (JJ)V */ -void Java_org_rocksdb_DBOptions_setBgerrorResumeRetryInterval( +void Java_org_forstdb_DBOptions_setBgerrorResumeRetryInterval( JNIEnv*, jclass, jlong jhandle, jlong jbgerror_resume_retry_interval) { auto* opt = reinterpret_cast(jhandle); opt->bgerror_resume_retry_interval = @@ -7808,11 +7855,11 @@ void Java_org_rocksdb_DBOptions_setBgerrorResumeRetryInterval( } /* - * Class: org_rocksdb_DBOptions + * Class: org_forstdb_DBOptions * Method: bgerrorResumeRetryInterval * Signature: (J)J */ -jlong Java_org_rocksdb_DBOptions_bgerrorResumeRetryInterval(JNIEnv*, jclass, +jlong Java_org_forstdb_DBOptions_bgerrorResumeRetryInterval(JNIEnv*, jclass, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->bgerror_resume_retry_interval); @@ -7822,21 +7869,21 @@ jlong Java_org_rocksdb_DBOptions_bgerrorResumeRetryInterval(JNIEnv*, jclass, // ROCKSDB_NAMESPACE::WriteOptions /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: newWriteOptions * Signature: ()J */ -jlong Java_org_rocksdb_WriteOptions_newWriteOptions(JNIEnv*, jclass) { +jlong Java_org_forstdb_WriteOptions_newWriteOptions(JNIEnv*, jclass) { auto* op = new ROCKSDB_NAMESPACE::WriteOptions(); return GET_CPLUSPLUS_POINTER(op); } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: copyWriteOptions * Signature: (J)J */ -jlong Java_org_rocksdb_WriteOptions_copyWriteOptions(JNIEnv*, jclass, +jlong Java_org_forstdb_WriteOptions_copyWriteOptions(JNIEnv*, jclass, jlong jhandle) { auto new_opt = new ROCKSDB_NAMESPACE::WriteOptions( *(reinterpret_cast(jhandle))); @@ -7844,11 +7891,11 @@ jlong Java_org_rocksdb_WriteOptions_copyWriteOptions(JNIEnv*, jclass, } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: disposeInternal * Signature: ()V */ -void Java_org_rocksdb_WriteOptions_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_WriteOptions_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* write_options = reinterpret_cast(jhandle); @@ -7857,30 +7904,30 @@ void Java_org_rocksdb_WriteOptions_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: setSync * Signature: (JZ)V */ -void Java_org_rocksdb_WriteOptions_setSync(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_WriteOptions_setSync(JNIEnv*, jobject, jlong jhandle, jboolean jflag) { reinterpret_cast(jhandle)->sync = jflag; } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: sync * Signature: (J)Z */ -jboolean Java_org_rocksdb_WriteOptions_sync(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_WriteOptions_sync(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->sync; } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: setDisableWAL * Signature: (JZ)V */ -void Java_org_rocksdb_WriteOptions_setDisableWAL(JNIEnv*, jobject, +void Java_org_forstdb_WriteOptions_setDisableWAL(JNIEnv*, jobject, jlong jhandle, jboolean jflag) { reinterpret_cast(jhandle)->disableWAL = @@ -7888,22 +7935,22 @@ void Java_org_rocksdb_WriteOptions_setDisableWAL(JNIEnv*, jobject, } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: disableWAL * Signature: (J)Z */ -jboolean Java_org_rocksdb_WriteOptions_disableWAL(JNIEnv*, jobject, +jboolean Java_org_forstdb_WriteOptions_disableWAL(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->disableWAL; } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: setIgnoreMissingColumnFamilies * Signature: (JZ)V */ -void Java_org_rocksdb_WriteOptions_setIgnoreMissingColumnFamilies( +void Java_org_forstdb_WriteOptions_setIgnoreMissingColumnFamilies( JNIEnv*, jobject, jlong jhandle, jboolean jignore_missing_column_families) { reinterpret_cast(jhandle) ->ignore_missing_column_families = @@ -7911,22 +7958,22 @@ void Java_org_rocksdb_WriteOptions_setIgnoreMissingColumnFamilies( } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: ignoreMissingColumnFamilies * Signature: (J)Z */ -jboolean Java_org_rocksdb_WriteOptions_ignoreMissingColumnFamilies( +jboolean Java_org_forstdb_WriteOptions_ignoreMissingColumnFamilies( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->ignore_missing_column_families; } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: setNoSlowdown * Signature: (JZ)V */ -void Java_org_rocksdb_WriteOptions_setNoSlowdown(JNIEnv*, jobject, +void Java_org_forstdb_WriteOptions_setNoSlowdown(JNIEnv*, jobject, jlong jhandle, jboolean jno_slowdown) { reinterpret_cast(jhandle)->no_slowdown = @@ -7934,53 +7981,53 @@ void Java_org_rocksdb_WriteOptions_setNoSlowdown(JNIEnv*, jobject, } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: noSlowdown * Signature: (J)Z */ -jboolean Java_org_rocksdb_WriteOptions_noSlowdown(JNIEnv*, jobject, +jboolean Java_org_forstdb_WriteOptions_noSlowdown(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->no_slowdown; } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: setLowPri * Signature: (JZ)V */ -void Java_org_rocksdb_WriteOptions_setLowPri(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_WriteOptions_setLowPri(JNIEnv*, jobject, jlong jhandle, jboolean jlow_pri) { reinterpret_cast(jhandle)->low_pri = static_cast(jlow_pri); } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: lowPri * Signature: (J)Z */ -jboolean Java_org_rocksdb_WriteOptions_lowPri(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_WriteOptions_lowPri(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->low_pri; } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: memtableInsertHintPerBatch * Signature: (J)Z */ -jboolean Java_org_rocksdb_WriteOptions_memtableInsertHintPerBatch( +jboolean Java_org_forstdb_WriteOptions_memtableInsertHintPerBatch( JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->memtable_insert_hint_per_batch; } /* - * Class: org_rocksdb_WriteOptions + * Class: org_forstdb_WriteOptions * Method: setMemtableInsertHintPerBatch * Signature: (JZ)V */ -void Java_org_rocksdb_WriteOptions_setMemtableInsertHintPerBatch( +void Java_org_forstdb_WriteOptions_setMemtableInsertHintPerBatch( JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_insert_hint_per_batch) { reinterpret_cast(jhandle) ->memtable_insert_hint_per_batch = @@ -7991,21 +8038,21 @@ void Java_org_rocksdb_WriteOptions_setMemtableInsertHintPerBatch( // ROCKSDB_NAMESPACE::ReadOptions /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: newReadOptions * Signature: ()J */ -jlong Java_org_rocksdb_ReadOptions_newReadOptions__(JNIEnv*, jclass) { +jlong Java_org_forstdb_ReadOptions_newReadOptions__(JNIEnv*, jclass) { auto* read_options = new ROCKSDB_NAMESPACE::ReadOptions(); return GET_CPLUSPLUS_POINTER(read_options); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: newReadOptions * Signature: (ZZ)J */ -jlong Java_org_rocksdb_ReadOptions_newReadOptions__ZZ( +jlong Java_org_forstdb_ReadOptions_newReadOptions__ZZ( JNIEnv*, jclass, jboolean jverify_checksums, jboolean jfill_cache) { auto* read_options = new ROCKSDB_NAMESPACE::ReadOptions( static_cast(jverify_checksums), static_cast(jfill_cache)); @@ -8013,11 +8060,11 @@ jlong Java_org_rocksdb_ReadOptions_newReadOptions__ZZ( } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: copyReadOptions * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_copyReadOptions(JNIEnv*, jclass, +jlong Java_org_forstdb_ReadOptions_copyReadOptions(JNIEnv*, jclass, jlong jhandle) { auto new_opt = new ROCKSDB_NAMESPACE::ReadOptions( *(reinterpret_cast(jhandle))); @@ -8025,11 +8072,11 @@ jlong Java_org_rocksdb_ReadOptions_copyReadOptions(JNIEnv*, jclass, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_ReadOptions_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_ReadOptions_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* read_options = reinterpret_cast(jhandle); @@ -8038,169 +8085,169 @@ void Java_org_rocksdb_ReadOptions_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setVerifyChecksums * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setVerifyChecksums( +void Java_org_forstdb_ReadOptions_setVerifyChecksums( JNIEnv*, jobject, jlong jhandle, jboolean jverify_checksums) { reinterpret_cast(jhandle)->verify_checksums = static_cast(jverify_checksums); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: verifyChecksums * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_verifyChecksums(JNIEnv*, jobject, +jboolean Java_org_forstdb_ReadOptions_verifyChecksums(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->verify_checksums; } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setFillCache * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setFillCache(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setFillCache(JNIEnv*, jobject, jlong jhandle, jboolean jfill_cache) { reinterpret_cast(jhandle)->fill_cache = static_cast(jfill_cache); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: fillCache * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_fillCache(JNIEnv*, jobject, +jboolean Java_org_forstdb_ReadOptions_fillCache(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->fill_cache; } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setTailing * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setTailing(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setTailing(JNIEnv*, jobject, jlong jhandle, jboolean jtailing) { reinterpret_cast(jhandle)->tailing = static_cast(jtailing); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: tailing * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_tailing(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_ReadOptions_tailing(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->tailing; } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: managed * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_managed(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_ReadOptions_managed(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->managed; } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setManaged * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setManaged(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setManaged(JNIEnv*, jobject, jlong jhandle, jboolean jmanaged) { reinterpret_cast(jhandle)->managed = static_cast(jmanaged); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: totalOrderSeek * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_totalOrderSeek(JNIEnv*, jobject, +jboolean Java_org_forstdb_ReadOptions_totalOrderSeek(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->total_order_seek; } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setTotalOrderSeek * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setTotalOrderSeek( +void Java_org_forstdb_ReadOptions_setTotalOrderSeek( JNIEnv*, jobject, jlong jhandle, jboolean jtotal_order_seek) { reinterpret_cast(jhandle)->total_order_seek = static_cast(jtotal_order_seek); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: prefixSameAsStart * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_prefixSameAsStart(JNIEnv*, jobject, +jboolean Java_org_forstdb_ReadOptions_prefixSameAsStart(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle) ->prefix_same_as_start; } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setPrefixSameAsStart * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setPrefixSameAsStart( +void Java_org_forstdb_ReadOptions_setPrefixSameAsStart( JNIEnv*, jobject, jlong jhandle, jboolean jprefix_same_as_start) { reinterpret_cast(jhandle) ->prefix_same_as_start = static_cast(jprefix_same_as_start); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: pinData * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_pinData(JNIEnv*, jobject, jlong jhandle) { +jboolean Java_org_forstdb_ReadOptions_pinData(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->pin_data; } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setPinData * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setPinData(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setPinData(JNIEnv*, jobject, jlong jhandle, jboolean jpin_data) { reinterpret_cast(jhandle)->pin_data = static_cast(jpin_data); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: backgroundPurgeOnIteratorCleanup * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_backgroundPurgeOnIteratorCleanup( +jboolean Java_org_forstdb_ReadOptions_backgroundPurgeOnIteratorCleanup( JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->background_purge_on_iterator_cleanup); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setBackgroundPurgeOnIteratorCleanup * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setBackgroundPurgeOnIteratorCleanup( +void Java_org_forstdb_ReadOptions_setBackgroundPurgeOnIteratorCleanup( JNIEnv*, jobject, jlong jhandle, jboolean jbackground_purge_on_iterator_cleanup) { auto* opt = reinterpret_cast(jhandle); @@ -8209,22 +8256,22 @@ void Java_org_rocksdb_ReadOptions_setBackgroundPurgeOnIteratorCleanup( } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: readaheadSize * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_readaheadSize(JNIEnv*, jobject, +jlong Java_org_forstdb_ReadOptions_readaheadSize(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->readahead_size); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setReadaheadSize * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setReadaheadSize(JNIEnv*, jobject, +void Java_org_forstdb_ReadOptions_setReadaheadSize(JNIEnv*, jobject, jlong jhandle, jlong jreadahead_size) { auto* opt = reinterpret_cast(jhandle); @@ -8232,22 +8279,22 @@ void Java_org_rocksdb_ReadOptions_setReadaheadSize(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: maxSkippableInternalKeys * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_maxSkippableInternalKeys(JNIEnv*, jobject, +jlong Java_org_forstdb_ReadOptions_maxSkippableInternalKeys(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->max_skippable_internal_keys); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setMaxSkippableInternalKeys * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setMaxSkippableInternalKeys( +void Java_org_forstdb_ReadOptions_setMaxSkippableInternalKeys( JNIEnv*, jobject, jlong jhandle, jlong jmax_skippable_internal_keys) { auto* opt = reinterpret_cast(jhandle); opt->max_skippable_internal_keys = @@ -8255,76 +8302,76 @@ void Java_org_rocksdb_ReadOptions_setMaxSkippableInternalKeys( } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: ignoreRangeDeletions * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_ignoreRangeDeletions(JNIEnv*, jobject, +jboolean Java_org_forstdb_ReadOptions_ignoreRangeDeletions(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->ignore_range_deletions); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setIgnoreRangeDeletions * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setIgnoreRangeDeletions( +void Java_org_forstdb_ReadOptions_setIgnoreRangeDeletions( JNIEnv*, jobject, jlong jhandle, jboolean jignore_range_deletions) { auto* opt = reinterpret_cast(jhandle); opt->ignore_range_deletions = static_cast(jignore_range_deletions); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setSnapshot * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setSnapshot(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setSnapshot(JNIEnv*, jobject, jlong jhandle, jlong jsnapshot) { reinterpret_cast(jhandle)->snapshot = reinterpret_cast(jsnapshot); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: snapshot * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_snapshot(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_ReadOptions_snapshot(JNIEnv*, jobject, jlong jhandle) { auto& snapshot = reinterpret_cast(jhandle)->snapshot; return GET_CPLUSPLUS_POINTER(snapshot); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: readTier * Signature: (J)B */ -jbyte Java_org_rocksdb_ReadOptions_readTier(JNIEnv*, jobject, jlong jhandle) { +jbyte Java_org_forstdb_ReadOptions_readTier(JNIEnv*, jobject, jlong jhandle) { return static_cast( reinterpret_cast(jhandle)->read_tier); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setReadTier * Signature: (JB)V */ -void Java_org_rocksdb_ReadOptions_setReadTier(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setReadTier(JNIEnv*, jobject, jlong jhandle, jbyte jread_tier) { reinterpret_cast(jhandle)->read_tier = static_cast(jread_tier); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setIterateUpperBound * Signature: (JJ)I */ -void Java_org_rocksdb_ReadOptions_setIterateUpperBound( +void Java_org_forstdb_ReadOptions_setIterateUpperBound( JNIEnv*, jobject, jlong jhandle, jlong jupper_bound_slice_handle) { reinterpret_cast(jhandle) ->iterate_upper_bound = @@ -8332,11 +8379,11 @@ void Java_org_rocksdb_ReadOptions_setIterateUpperBound( } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: iterateUpperBound * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_iterateUpperBound(JNIEnv*, jobject, +jlong Java_org_forstdb_ReadOptions_iterateUpperBound(JNIEnv*, jobject, jlong jhandle) { auto& upper_bound_slice_handle = reinterpret_cast(jhandle) @@ -8345,11 +8392,11 @@ jlong Java_org_rocksdb_ReadOptions_iterateUpperBound(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setIterateLowerBound * Signature: (JJ)I */ -void Java_org_rocksdb_ReadOptions_setIterateLowerBound( +void Java_org_forstdb_ReadOptions_setIterateLowerBound( JNIEnv*, jobject, jlong jhandle, jlong jlower_bound_slice_handle) { reinterpret_cast(jhandle) ->iterate_lower_bound = @@ -8357,11 +8404,11 @@ void Java_org_rocksdb_ReadOptions_setIterateLowerBound( } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: iterateLowerBound * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_iterateLowerBound(JNIEnv*, jobject, +jlong Java_org_forstdb_ReadOptions_iterateLowerBound(JNIEnv*, jobject, jlong jhandle) { auto& lower_bound_slice_handle = reinterpret_cast(jhandle) @@ -8370,11 +8417,11 @@ jlong Java_org_rocksdb_ReadOptions_iterateLowerBound(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setTableFilter * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setTableFilter( +void Java_org_forstdb_ReadOptions_setTableFilter( JNIEnv*, jobject, jlong jhandle, jlong jjni_table_filter_handle) { auto* opt = reinterpret_cast(jhandle); auto* jni_table_filter = @@ -8384,44 +8431,44 @@ void Java_org_rocksdb_ReadOptions_setTableFilter( } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: autoPrefixMode * Signature: (J)Z */ -jboolean Java_org_rocksdb_ReadOptions_autoPrefixMode(JNIEnv*, jobject, +jboolean Java_org_forstdb_ReadOptions_autoPrefixMode(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->auto_prefix_mode); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setAutoPrefixMode * Signature: (JZ)V */ -void Java_org_rocksdb_ReadOptions_setAutoPrefixMode( +void Java_org_forstdb_ReadOptions_setAutoPrefixMode( JNIEnv*, jobject, jlong jhandle, jboolean jauto_prefix_mode) { auto* opt = reinterpret_cast(jhandle); opt->auto_prefix_mode = static_cast(jauto_prefix_mode); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: timestamp * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_timestamp(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_ReadOptions_timestamp(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); auto& timestamp_slice_handle = opt->timestamp; return reinterpret_cast(timestamp_slice_handle); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setTimestamp * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setTimestamp(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setTimestamp(JNIEnv*, jobject, jlong jhandle, jlong jtimestamp_slice_handle) { auto* opt = reinterpret_cast(jhandle); opt->timestamp = @@ -8429,11 +8476,11 @@ void Java_org_rocksdb_ReadOptions_setTimestamp(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: iterStartTs * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_iterStartTs(JNIEnv*, jobject, +jlong Java_org_forstdb_ReadOptions_iterStartTs(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); auto& iter_start_ts_handle = opt->iter_start_ts; @@ -8441,11 +8488,11 @@ jlong Java_org_rocksdb_ReadOptions_iterStartTs(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setIterStartTs * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setIterStartTs(JNIEnv*, jobject, +void Java_org_forstdb_ReadOptions_setIterStartTs(JNIEnv*, jobject, jlong jhandle, jlong jiter_start_ts_handle) { auto* opt = reinterpret_cast(jhandle); @@ -8454,42 +8501,42 @@ void Java_org_rocksdb_ReadOptions_setIterStartTs(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: deadline * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_deadline(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_ReadOptions_deadline(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->deadline.count()); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setDeadline * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setDeadline(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setDeadline(JNIEnv*, jobject, jlong jhandle, jlong jdeadline) { auto* opt = reinterpret_cast(jhandle); opt->deadline = std::chrono::microseconds(static_cast(jdeadline)); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: ioTimeout * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_ioTimeout(JNIEnv*, jobject, jlong jhandle) { +jlong Java_org_forstdb_ReadOptions_ioTimeout(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->io_timeout.count()); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setIoTimeout * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setIoTimeout(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_ReadOptions_setIoTimeout(JNIEnv*, jobject, jlong jhandle, jlong jio_timeout) { auto* opt = reinterpret_cast(jhandle); opt->io_timeout = @@ -8497,22 +8544,22 @@ void Java_org_rocksdb_ReadOptions_setIoTimeout(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: valueSizeSofLimit * Signature: (J)J */ -jlong Java_org_rocksdb_ReadOptions_valueSizeSoftLimit(JNIEnv*, jobject, +jlong Java_org_forstdb_ReadOptions_valueSizeSoftLimit(JNIEnv*, jobject, jlong jhandle) { auto* opt = reinterpret_cast(jhandle); return static_cast(opt->value_size_soft_limit); } /* - * Class: org_rocksdb_ReadOptions + * Class: org_forstdb_ReadOptions * Method: setValueSizeSofLimit * Signature: (JJ)V */ -void Java_org_rocksdb_ReadOptions_setValueSizeSoftLimit( +void Java_org_forstdb_ReadOptions_setValueSizeSoftLimit( JNIEnv*, jobject, jlong jhandle, jlong jvalue_size_soft_limit) { auto* opt = reinterpret_cast(jhandle); opt->value_size_soft_limit = static_cast(jvalue_size_soft_limit); @@ -8522,21 +8569,21 @@ void Java_org_rocksdb_ReadOptions_setValueSizeSoftLimit( // ROCKSDB_NAMESPACE::ComparatorOptions /* - * Class: org_rocksdb_ComparatorOptions + * Class: org_forstdb_ComparatorOptions * Method: newComparatorOptions * Signature: ()J */ -jlong Java_org_rocksdb_ComparatorOptions_newComparatorOptions(JNIEnv*, jclass) { +jlong Java_org_forstdb_ComparatorOptions_newComparatorOptions(JNIEnv*, jclass) { auto* comparator_opt = new ROCKSDB_NAMESPACE::ComparatorJniCallbackOptions(); return GET_CPLUSPLUS_POINTER(comparator_opt); } /* - * Class: org_rocksdb_ComparatorOptions + * Class: org_forstdb_ComparatorOptions * Method: reusedSynchronisationType * Signature: (J)B */ -jbyte Java_org_rocksdb_ComparatorOptions_reusedSynchronisationType( +jbyte Java_org_forstdb_ComparatorOptions_reusedSynchronisationType( JNIEnv*, jobject, jlong jhandle) { auto* comparator_opt = reinterpret_cast( @@ -8547,11 +8594,11 @@ jbyte Java_org_rocksdb_ComparatorOptions_reusedSynchronisationType( } /* - * Class: org_rocksdb_ComparatorOptions + * Class: org_forstdb_ComparatorOptions * Method: setReusedSynchronisationType * Signature: (JB)V */ -void Java_org_rocksdb_ComparatorOptions_setReusedSynchronisationType( +void Java_org_forstdb_ComparatorOptions_setReusedSynchronisationType( JNIEnv*, jobject, jlong jhandle, jbyte jreused_synhcronisation_type) { auto* comparator_opt = reinterpret_cast( @@ -8562,11 +8609,11 @@ void Java_org_rocksdb_ComparatorOptions_setReusedSynchronisationType( } /* - * Class: org_rocksdb_ComparatorOptions + * Class: org_forstdb_ComparatorOptions * Method: useDirectBuffer * Signature: (J)Z */ -jboolean Java_org_rocksdb_ComparatorOptions_useDirectBuffer(JNIEnv*, jobject, +jboolean Java_org_forstdb_ComparatorOptions_useDirectBuffer(JNIEnv*, jobject, jlong jhandle) { return static_cast( reinterpret_cast( @@ -8575,22 +8622,22 @@ jboolean Java_org_rocksdb_ComparatorOptions_useDirectBuffer(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ComparatorOptions + * Class: org_forstdb_ComparatorOptions * Method: setUseDirectBuffer * Signature: (JZ)V */ -void Java_org_rocksdb_ComparatorOptions_setUseDirectBuffer( +void Java_org_forstdb_ComparatorOptions_setUseDirectBuffer( JNIEnv*, jobject, jlong jhandle, jboolean jdirect_buffer) { reinterpret_cast(jhandle) ->direct_buffer = jdirect_buffer == JNI_TRUE; } /* - * Class: org_rocksdb_ComparatorOptions + * Class: org_forstdb_ComparatorOptions * Method: maxReusedBufferSize * Signature: (J)I */ -jint Java_org_rocksdb_ComparatorOptions_maxReusedBufferSize(JNIEnv*, jobject, +jint Java_org_forstdb_ComparatorOptions_maxReusedBufferSize(JNIEnv*, jobject, jlong jhandle) { return static_cast( reinterpret_cast( @@ -8599,22 +8646,22 @@ jint Java_org_rocksdb_ComparatorOptions_maxReusedBufferSize(JNIEnv*, jobject, } /* - * Class: org_rocksdb_ComparatorOptions + * Class: org_forstdb_ComparatorOptions * Method: setMaxReusedBufferSize * Signature: (JI)V */ -void Java_org_rocksdb_ComparatorOptions_setMaxReusedBufferSize( +void Java_org_forstdb_ComparatorOptions_setMaxReusedBufferSize( JNIEnv*, jobject, jlong jhandle, jint jmax_reused_buffer_size) { reinterpret_cast(jhandle) ->max_reused_buffer_size = static_cast(jmax_reused_buffer_size); } /* - * Class: org_rocksdb_ComparatorOptions + * Class: org_forstdb_ComparatorOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_ComparatorOptions_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_ComparatorOptions_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* comparator_opt = reinterpret_cast( @@ -8627,21 +8674,21 @@ void Java_org_rocksdb_ComparatorOptions_disposeInternal(JNIEnv*, jobject, // ROCKSDB_NAMESPACE::FlushOptions /* - * Class: org_rocksdb_FlushOptions + * Class: org_forstdb_FlushOptions * Method: newFlushOptions * Signature: ()J */ -jlong Java_org_rocksdb_FlushOptions_newFlushOptions(JNIEnv*, jclass) { +jlong Java_org_forstdb_FlushOptions_newFlushOptions(JNIEnv*, jclass) { auto* flush_opt = new ROCKSDB_NAMESPACE::FlushOptions(); return GET_CPLUSPLUS_POINTER(flush_opt); } /* - * Class: org_rocksdb_FlushOptions + * Class: org_forstdb_FlushOptions * Method: setWaitForFlush * Signature: (JZ)V */ -void Java_org_rocksdb_FlushOptions_setWaitForFlush(JNIEnv*, jobject, +void Java_org_forstdb_FlushOptions_setWaitForFlush(JNIEnv*, jobject, jlong jhandle, jboolean jwait) { reinterpret_cast(jhandle)->wait = @@ -8649,21 +8696,21 @@ void Java_org_rocksdb_FlushOptions_setWaitForFlush(JNIEnv*, jobject, } /* - * Class: org_rocksdb_FlushOptions + * Class: org_forstdb_FlushOptions * Method: waitForFlush * Signature: (J)Z */ -jboolean Java_org_rocksdb_FlushOptions_waitForFlush(JNIEnv*, jobject, +jboolean Java_org_forstdb_FlushOptions_waitForFlush(JNIEnv*, jobject, jlong jhandle) { return reinterpret_cast(jhandle)->wait; } /* - * Class: org_rocksdb_FlushOptions + * Class: org_forstdb_FlushOptions * Method: setAllowWriteStall * Signature: (JZ)V */ -void Java_org_rocksdb_FlushOptions_setAllowWriteStall( +void Java_org_forstdb_FlushOptions_setAllowWriteStall( JNIEnv*, jobject, jlong jhandle, jboolean jallow_write_stall) { auto* flush_options = reinterpret_cast(jhandle); @@ -8671,11 +8718,11 @@ void Java_org_rocksdb_FlushOptions_setAllowWriteStall( } /* - * Class: org_rocksdb_FlushOptions + * Class: org_forstdb_FlushOptions * Method: allowWriteStall * Signature: (J)Z */ -jboolean Java_org_rocksdb_FlushOptions_allowWriteStall(JNIEnv*, jobject, +jboolean Java_org_forstdb_FlushOptions_allowWriteStall(JNIEnv*, jobject, jlong jhandle) { auto* flush_options = reinterpret_cast(jhandle); @@ -8683,11 +8730,11 @@ jboolean Java_org_rocksdb_FlushOptions_allowWriteStall(JNIEnv*, jobject, } /* - * Class: org_rocksdb_FlushOptions + * Class: org_forstdb_FlushOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_FlushOptions_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_FlushOptions_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* flush_opt = reinterpret_cast(jhandle); assert(flush_opt != nullptr); diff --git a/java/rocksjni/options_util.cc b/java/forstjni/options_util.cc similarity index 66% rename from java/rocksjni/options_util.cc rename to java/forstjni/options_util.cc index c3d7fcef6..8074c8411 100644 --- a/java/rocksjni/options_util.cc +++ b/java/forstjni/options_util.cc @@ -12,10 +12,10 @@ #include -#include "include/org_rocksdb_OptionsUtil.h" +#include "include/org_forstdb_OptionsUtil.h" #include "rocksdb/db.h" #include "rocksdb/env.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" void build_column_family_descriptor_list( JNIEnv* env, jobject jcfds, @@ -52,11 +52,11 @@ void build_column_family_descriptor_list( } /* - * Class: org_rocksdb_OptionsUtil + * Class: org_forstdb_OptionsUtil * Method: loadLatestOptions * Signature: (JLjava/lang/String;JLjava/util/List;)V */ -void Java_org_rocksdb_OptionsUtil_loadLatestOptions( +void Java_org_forstdb_OptionsUtil_loadLatestOptions( JNIEnv* env, jclass /*jcls*/, jlong cfg_handle, jstring jdbpath, jlong jdb_opts_handle, jobject jcfds) { jboolean has_exception = JNI_FALSE; @@ -82,11 +82,11 @@ void Java_org_rocksdb_OptionsUtil_loadLatestOptions( } /* - * Class: org_rocksdb_OptionsUtil + * Class: org_forstdb_OptionsUtil * Method: loadOptionsFromFile * Signature: (JLjava/lang/String;JLjava/util/List;)V */ -void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile( +void Java_org_forstdb_OptionsUtil_loadOptionsFromFile( JNIEnv* env, jclass /*jcls*/, jlong cfg_handle, jstring jopts_file_name, jlong jdb_opts_handle, jobject jcfds) { jboolean has_exception = JNI_FALSE; @@ -112,11 +112,11 @@ void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile( } /* - * Class: org_rocksdb_OptionsUtil + * Class: org_forstdb_OptionsUtil * Method: getLatestOptionsFileName * Signature: (Ljava/lang/String;J)Ljava/lang/String; */ -jstring Java_org_rocksdb_OptionsUtil_getLatestOptionsFileName( +jstring Java_org_forstdb_OptionsUtil_getLatestOptionsFileName( JNIEnv* env, jclass /*jcls*/, jstring jdbpath, jlong jenv_handle) { jboolean has_exception = JNI_FALSE; auto db_path = @@ -137,3 +137,51 @@ jstring Java_org_rocksdb_OptionsUtil_getLatestOptionsFileName( return env->NewStringUTF(options_file_name.c_str()); } } + +/* + * Class: org_forstdb_OptionsUtil + * Method: readTableFormatConfig + * Signature: (J)Lorg/forstdb/TableFormatConfig; + */ +jobject Java_org_forstdb_OptionsUtil_readTableFormatConfig(JNIEnv* env, jclass, + jlong jcf_options) { + if (jcf_options == 0) { + env->ThrowNew( + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::getJClass(env), + "Null column family options handle supplied to " + "readNewTableFormatConfig"); + return nullptr; + } + + auto* cf_options = + reinterpret_cast(jcf_options); + auto* table_factory = cf_options->table_factory.get(); + if (table_factory == nullptr) { + env->ThrowNew( + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::getJClass(env), + "Column family options supplied to readNewTableFormatConfig has no " + "table options"); + return nullptr; + } + + if (strcmp(ROCKSDB_NAMESPACE::TableFactory::kBlockBasedTableName(), + table_factory->Name()) == 0) { + auto* table_factory_options = + table_factory->GetOptions(); + if (table_factory_options == nullptr) { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( + env, std::string("Null table format options supplied to " + "readNewTableFormatConfig() ") + + table_factory->Name()); + return nullptr; + } + return ROCKSDB_NAMESPACE::BlockBasedTableOptionsJni::construct( + env, table_factory_options); + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( + env, std::string("readNewTableFormatConfig() is not implemented for " + "this table format: ") + + table_factory->Name()); + return nullptr; + } +} diff --git a/java/rocksjni/persistent_cache.cc b/java/forstjni/persistent_cache.cc similarity index 85% rename from java/rocksjni/persistent_cache.cc rename to java/forstjni/persistent_cache.cc index 295d91798..f9a650751 100644 --- a/java/rocksjni/persistent_cache.cc +++ b/java/forstjni/persistent_cache.cc @@ -12,17 +12,17 @@ #include -#include "include/org_rocksdb_PersistentCache.h" +#include "include/org_forstdb_PersistentCache.h" #include "loggerjnicallback.h" #include "portal.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_PersistentCache + * Class: org_forstdb_PersistentCache * Method: newPersistentCache * Signature: (JLjava/lang/String;JJZ)J */ -jlong Java_org_rocksdb_PersistentCache_newPersistentCache( +jlong Java_org_forstdb_PersistentCache_newPersistentCache( JNIEnv* env, jclass, jlong jenv_handle, jstring jpath, jlong jsz, jlong jlogger_handle, jboolean joptimized_for_nvm) { auto* rocks_env = reinterpret_cast(jenv_handle); @@ -47,11 +47,11 @@ jlong Java_org_rocksdb_PersistentCache_newPersistentCache( } /* - * Class: org_rocksdb_PersistentCache + * Class: org_forstdb_PersistentCache * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_PersistentCache_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_PersistentCache_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* cache = reinterpret_cast*>( diff --git a/java/rocksjni/portal.h b/java/forstjni/portal.h similarity index 94% rename from java/rocksjni/portal.h rename to java/forstjni/portal.h index fc9763195..0e67a7691 100644 --- a/java/rocksjni/portal.h +++ b/java/forstjni/portal.h @@ -27,6 +27,7 @@ #include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/filter_policy.h" +#include "rocksdb/perf_level.h" #include "rocksdb/rate_limiter.h" #include "rocksdb/status.h" #include "rocksdb/table.h" @@ -34,16 +35,16 @@ #include "rocksdb/utilities/memory_util.h" #include "rocksdb/utilities/transaction_db.h" #include "rocksdb/utilities/write_batch_with_index.h" -#include "rocksjni/compaction_filter_factory_jnicallback.h" -#include "rocksjni/comparatorjnicallback.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/event_listener_jnicallback.h" -#include "rocksjni/loggerjnicallback.h" -#include "rocksjni/table_filter_jnicallback.h" -#include "rocksjni/trace_writer_jnicallback.h" -#include "rocksjni/transaction_notifier_jnicallback.h" -#include "rocksjni/wal_filter_jnicallback.h" -#include "rocksjni/writebatchhandlerjnicallback.h" +#include "forstjni/compaction_filter_factory_jnicallback.h" +#include "forstjni/comparatorjnicallback.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/event_listener_jnicallback.h" +#include "forstjni/loggerjnicallback.h" +#include "forstjni/table_filter_jnicallback.h" +#include "forstjni/trace_writer_jnicallback.h" +#include "forstjni/transaction_notifier_jnicallback.h" +#include "forstjni/wal_filter_jnicallback.h" +#include "forstjni/writebatchhandlerjnicallback.h" // Remove macro on windows #ifdef DELETE @@ -207,6 +208,18 @@ class IllegalArgumentExceptionJni return JavaException::ThrowNew(env, s.ToString()); } + + /** + * Create and throw a Java IllegalArgumentException with the provided message + * + * @param env A pointer to the Java environment + * @param msg The message for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const std::string& msg) { + return JavaException::ThrowNew(env, msg); + } }; // The portal class for org.rocksdb.Status.Code @@ -222,7 +235,7 @@ class CodeJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/Status$Code"); + return JavaClass::getJClass(env, "org/forstdb/Status$Code"); } /** @@ -259,7 +272,7 @@ class SubCodeJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/Status$SubCode"); + return JavaClass::getJClass(env, "org/forstdb/Status$SubCode"); } /** @@ -323,7 +336,7 @@ class StatusJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/Status"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/Status"); } /** @@ -342,7 +355,7 @@ class StatusJni } static jmethodID mid = - env->GetMethodID(jclazz, "getCode", "()Lorg/rocksdb/Status$Code;"); + env->GetMethodID(jclazz, "getCode", "()Lorg/forstdb/Status$Code;"); assert(mid != nullptr); return mid; } @@ -363,7 +376,7 @@ class StatusJni } static jmethodID mid = env->GetMethodID(jclazz, "getSubCode", - "()Lorg/rocksdb/Status$SubCode;"); + "()Lorg/forstdb/Status$SubCode;"); assert(mid != nullptr); return mid; } @@ -732,7 +745,7 @@ class RocksDBExceptionJni : public JavaException { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaException::getJClass(env, "org/rocksdb/RocksDBException"); + return JavaException::getJClass(env, "org/forstdb/RocksDBException"); } /** @@ -788,7 +801,7 @@ class RocksDBExceptionJni : public JavaException { // get the constructor of org.rocksdb.RocksDBException jmethodID mid = - env->GetMethodID(jclazz, "", "(Lorg/rocksdb/Status;)V"); + env->GetMethodID(jclazz, "", "(Lorg/forstdb/Status;)V"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError std::cerr @@ -879,7 +892,7 @@ class RocksDBExceptionJni : public JavaException { // get the constructor of org.rocksdb.RocksDBException jmethodID mid = env->GetMethodID( - jclazz, "", "(Ljava/lang/String;Lorg/rocksdb/Status;)V"); + jclazz, "", "(Ljava/lang/String;Lorg/forstdb/Status;)V"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError std::cerr @@ -977,7 +990,7 @@ class RocksDBExceptionJni : public JavaException { } static jmethodID mid = - env->GetMethodID(jclazz, "getStatus", "()Lorg/rocksdb/Status;"); + env->GetMethodID(jclazz, "getStatus", "()Lorg/forstdb/Status;"); assert(mid != nullptr); return mid; } @@ -2797,7 +2810,7 @@ class RocksDBJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/RocksDB"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/RocksDB"); } }; @@ -2815,7 +2828,7 @@ class OptionsJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/Options"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/Options"); } }; @@ -2833,7 +2846,7 @@ class DBOptionsJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/DBOptions"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/DBOptions"); } }; @@ -2853,7 +2866,7 @@ class ColumnFamilyOptionsJni */ static jclass getJClass(JNIEnv* env) { return RocksDBNativeClass::getJClass(env, - "org/rocksdb/ColumnFamilyOptions"); + "org/forstdb/ColumnFamilyOptions"); } /** @@ -2905,7 +2918,7 @@ class WriteOptionsJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/WriteOptions"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/WriteOptions"); } }; @@ -2924,7 +2937,7 @@ class ReadOptionsJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/ReadOptions"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/ReadOptions"); } }; @@ -2942,7 +2955,7 @@ class WriteBatchJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/WriteBatch"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/WriteBatch"); } /** @@ -2992,7 +3005,7 @@ class WriteBatchHandlerJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/WriteBatch$Handler"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/WriteBatch$Handler"); } /** @@ -3389,7 +3402,7 @@ class WriteBatchSavePointJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/WriteBatch$SavePoint"); + return JavaClass::getJClass(env, "org/forstdb/WriteBatch$SavePoint"); } /** @@ -3463,7 +3476,7 @@ class WriteBatchWithIndexJni */ static jclass getJClass(JNIEnv* env) { return RocksDBNativeClass::getJClass(env, - "org/rocksdb/WriteBatchWithIndex"); + "org/forstdb/WriteBatchWithIndex"); } }; @@ -3480,7 +3493,7 @@ class HistogramDataJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/HistogramData"); + return JavaClass::getJClass(env, "org/forstdb/HistogramData"); } /** @@ -3520,7 +3533,7 @@ class BackupEngineOptionsJni */ static jclass getJClass(JNIEnv* env) { return RocksDBNativeClass::getJClass(env, - "org/rocksdb/BackupEngineOptions"); + "org/forstdb/BackupEngineOptions"); } }; @@ -3539,7 +3552,7 @@ class BackupEngineJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/BackupEngine"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/BackupEngine"); } }; @@ -3557,17 +3570,24 @@ class IteratorJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/RocksIterator"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/RocksIterator"); } }; -// The portal class for org.rocksdb.Filter -class FilterJni +// The portal class for org.rocksdb.FilterPolicy + +enum FilterPolicyTypeJni { + kUnknownFilterPolicy = 0x00, + kBloomFilterPolicy = 0x01, + kRibbonFilterPolicy = 0x02, +}; +class FilterPolicyJni : public RocksDBNativeClass< - std::shared_ptr*, FilterJni> { + std::shared_ptr*, FilterPolicyJni> { + private: public: /** - * Get the Java Class org.rocksdb.Filter + * Get the Java Class org.rocksdb.FilterPolicy * * @param env A pointer to the Java environment * @@ -3576,7 +3596,19 @@ class FilterJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/Filter"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/FilterPolicy"); + } + + static jbyte toJavaIndexType(const FilterPolicyTypeJni& filter_policy_type) { + return static_cast(filter_policy_type); + } + + static FilterPolicyTypeJni getFilterPolicyType( + const std::string& policy_class_name) { + if (policy_class_name == "rocksdb.BuiltinBloomFilter") { + return kBloomFilterPolicy; + } + return kUnknownFilterPolicy; } }; @@ -3608,7 +3640,7 @@ class ColumnFamilyHandleJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/ColumnFamilyHandle"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/ColumnFamilyHandle"); } }; @@ -3627,7 +3659,7 @@ class FlushOptionsJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/FlushOptions"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/FlushOptions"); } }; @@ -3647,7 +3679,7 @@ class ComparatorOptionsJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/ComparatorOptions"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/ComparatorOptions"); } }; @@ -3668,7 +3700,7 @@ class AbstractCompactionFilterFactoryJni */ static jclass getJClass(JNIEnv* env) { return RocksDBNativeClass::getJClass( - env, "org/rocksdb/AbstractCompactionFilterFactory"); + env, "org/forstdb/AbstractCompactionFilterFactory"); } /** @@ -3722,7 +3754,7 @@ class AbstractTransactionNotifierJni public: static jclass getJClass(JNIEnv* env) { return RocksDBNativeClass::getJClass( - env, "org/rocksdb/AbstractTransactionNotifier"); + env, "org/forstdb/AbstractTransactionNotifier"); } // Get the java method `snapshotCreated` @@ -3753,7 +3785,7 @@ class AbstractComparatorJniBridge : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/AbstractComparatorJniBridge"); + return JavaClass::getJClass(env, "org/forstdb/AbstractComparatorJniBridge"); } /** @@ -3768,7 +3800,7 @@ class AbstractComparatorJniBridge : public JavaClass { static jmethodID getCompareInternalMethodId(JNIEnv* env, jclass jclazz) { static jmethodID mid = env->GetStaticMethodID(jclazz, "compareInternal", - "(Lorg/rocksdb/AbstractComparator;Ljava/nio/" + "(Lorg/forstdb/AbstractComparator;Ljava/nio/" "ByteBuffer;ILjava/nio/ByteBuffer;I)I"); assert(mid != nullptr); return mid; @@ -3787,7 +3819,7 @@ class AbstractComparatorJniBridge : public JavaClass { jclass jclazz) { static jmethodID mid = env->GetStaticMethodID(jclazz, "findShortestSeparatorInternal", - "(Lorg/rocksdb/AbstractComparator;Ljava/nio/" + "(Lorg/forstdb/AbstractComparator;Ljava/nio/" "ByteBuffer;ILjava/nio/ByteBuffer;I)I"); assert(mid != nullptr); return mid; @@ -3806,7 +3838,7 @@ class AbstractComparatorJniBridge : public JavaClass { jclass jclazz) { static jmethodID mid = env->GetStaticMethodID( jclazz, "findShortSuccessorInternal", - "(Lorg/rocksdb/AbstractComparator;Ljava/nio/ByteBuffer;I)I"); + "(Lorg/forstdb/AbstractComparator;Ljava/nio/ByteBuffer;I)I"); assert(mid != nullptr); return mid; } @@ -3827,7 +3859,7 @@ class AbstractComparatorJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/AbstractComparator"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/AbstractComparator"); } /** @@ -3867,7 +3899,7 @@ class AbstractSliceJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/AbstractSlice"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/AbstractSlice"); } }; @@ -3886,7 +3918,7 @@ class SliceJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/Slice"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/Slice"); } /** @@ -3934,7 +3966,7 @@ class DirectSliceJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/DirectSlice"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/DirectSlice"); } /** @@ -3980,7 +4012,7 @@ class BackupInfoJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/BackupInfo"); + return JavaClass::getJClass(env, "org/forstdb/BackupInfo"); } /** @@ -4124,7 +4156,7 @@ class WBWIRocksIteratorJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/WBWIRocksIterator"); + return JavaClass::getJClass(env, "org/forstdb/WBWIRocksIterator"); } /** @@ -4143,7 +4175,7 @@ class WBWIRocksIteratorJni : public JavaClass { } static jfieldID fid = env->GetFieldID( - jclazz, "entry", "Lorg/rocksdb/WBWIRocksIterator$WriteEntry;"); + jclazz, "entry", "Lorg/forstdb/WBWIRocksIterator$WriteEntry;"); assert(fid != nullptr); return fid; } @@ -4249,7 +4281,7 @@ class WriteTypeJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/WBWIRocksIterator$WriteType"); + return JavaClass::getJClass(env, "org/forstdb/WBWIRocksIterator$WriteType"); } /** @@ -4269,7 +4301,7 @@ class WriteTypeJni : public JavaClass { } jfieldID jfid = env->GetStaticFieldID( - jclazz, name, "Lorg/rocksdb/WBWIRocksIterator$WriteType;"); + jclazz, name, "Lorg/forstdb/WBWIRocksIterator$WriteType;"); if (env->ExceptionCheck()) { // exception occurred while getting field return nullptr; @@ -4297,7 +4329,7 @@ class WriteEntryJni : public JavaClass { */ static jclass getJClass(JNIEnv* env) { return JavaClass::getJClass(env, - "org/rocksdb/WBWIRocksIterator$WriteEntry"); + "org/forstdb/WBWIRocksIterator$WriteEntry"); } }; @@ -4383,7 +4415,7 @@ class InfoLogLevelJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/InfoLogLevel"); + return JavaClass::getJClass(env, "org/forstdb/InfoLogLevel"); } /** @@ -4403,7 +4435,7 @@ class InfoLogLevelJni : public JavaClass { } jfieldID jfid = - env->GetStaticFieldID(jclazz, name, "Lorg/rocksdb/InfoLogLevel;"); + env->GetStaticFieldID(jclazz, name, "Lorg/forstdb/InfoLogLevel;"); if (env->ExceptionCheck()) { // exception occurred while getting field return nullptr; @@ -4423,7 +4455,7 @@ class LoggerJni std::shared_ptr*, LoggerJni> { public: /** - * Get the Java Class org/rocksdb/Logger + * Get the Java Class org/forstdb/Logger * * @param env A pointer to the Java environment * @@ -4432,7 +4464,7 @@ class LoggerJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/Logger"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/Logger"); } /** @@ -4451,7 +4483,7 @@ class LoggerJni } static jmethodID mid = env->GetMethodID( - jclazz, "log", "(Lorg/rocksdb/InfoLogLevel;Ljava/lang/String;)V"); + jclazz, "log", "(Lorg/forstdb/InfoLogLevel;Ljava/lang/String;)V"); assert(mid != nullptr); return mid; } @@ -4471,7 +4503,7 @@ class BatchResultJni : public JavaClass { */ static jclass getJClass(JNIEnv* env) { return JavaClass::getJClass( - env, "org/rocksdb/TransactionLogIterator$BatchResult"); + env, "org/forstdb/TransactionLogIterator$BatchResult"); } /** @@ -5131,6 +5163,27 @@ class TickerTypeJni { return -0x3B; case ROCKSDB_NAMESPACE::Tickers::BLOCK_CHECKSUM_MISMATCH_COUNT: return -0x3C; + case ROCKSDB_NAMESPACE::Tickers::READAHEAD_TRIMMED: + return -0x3D; + case ROCKSDB_NAMESPACE::Tickers::FIFO_MAX_SIZE_COMPACTIONS: + return -0x3E; + case ROCKSDB_NAMESPACE::Tickers::FIFO_TTL_COMPACTIONS: + return -0x3F; + case ROCKSDB_NAMESPACE::Tickers::PREFETCH_BYTES: + return -0x40; + case ROCKSDB_NAMESPACE::Tickers::PREFETCH_BYTES_USEFUL: + return -0x41; + case ROCKSDB_NAMESPACE::Tickers::PREFETCH_HITS: + return -0x42; + case ROCKSDB_NAMESPACE::Tickers::COMPRESSED_SECONDARY_CACHE_DUMMY_HITS: + return -0x43; + case ROCKSDB_NAMESPACE::Tickers::COMPRESSED_SECONDARY_CACHE_HITS: + return -0x44; + case ROCKSDB_NAMESPACE::Tickers::COMPRESSED_SECONDARY_CACHE_PROMOTIONS: + return -0x45; + case ROCKSDB_NAMESPACE::Tickers:: + COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS: + return -0x46; case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX: // 0x5F was the max value in the initial copy of tickers to Java. // Since these values are exposed directly to Java clients, we keep @@ -5494,6 +5547,29 @@ class TickerTypeJni { return ROCKSDB_NAMESPACE::Tickers::TABLE_OPEN_PREFETCH_TAIL_HIT; case -0x3C: return ROCKSDB_NAMESPACE::Tickers::BLOCK_CHECKSUM_MISMATCH_COUNT; + case -0x3D: + return ROCKSDB_NAMESPACE::Tickers::READAHEAD_TRIMMED; + case -0x3E: + return ROCKSDB_NAMESPACE::Tickers::FIFO_MAX_SIZE_COMPACTIONS; + case -0x3F: + return ROCKSDB_NAMESPACE::Tickers::FIFO_TTL_COMPACTIONS; + case -0x40: + return ROCKSDB_NAMESPACE::Tickers::PREFETCH_BYTES; + case -0x41: + return ROCKSDB_NAMESPACE::Tickers::PREFETCH_BYTES_USEFUL; + case -0x42: + return ROCKSDB_NAMESPACE::Tickers::PREFETCH_HITS; + case -0x43: + return ROCKSDB_NAMESPACE::Tickers:: + COMPRESSED_SECONDARY_CACHE_DUMMY_HITS; + case -0x44: + return ROCKSDB_NAMESPACE::Tickers::COMPRESSED_SECONDARY_CACHE_HITS; + case -0x45: + return ROCKSDB_NAMESPACE::Tickers:: + COMPRESSED_SECONDARY_CACHE_PROMOTIONS; + case -0x46: + return ROCKSDB_NAMESPACE::Tickers:: + COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS; case 0x5F: // 0x5F was the max value in the initial copy of tickers to Java. // Since these values are exposed directly to Java clients, we keep @@ -5629,6 +5705,17 @@ class HistogramTypeJni { return 0x3B; case ROCKSDB_NAMESPACE::Histograms::FILE_READ_DB_OPEN_MICROS: return 0x3C; + case ROCKSDB_NAMESPACE::Histograms::FILE_READ_GET_MICROS: + return 0x3D; + case ROCKSDB_NAMESPACE::Histograms::FILE_READ_MULTIGET_MICROS: + return 0x3E; + case ROCKSDB_NAMESPACE::Histograms::FILE_READ_DB_ITERATOR_MICROS: + return 0x3F; + case ROCKSDB_NAMESPACE::Histograms::FILE_READ_VERIFY_DB_CHECKSUM_MICROS: + return 0x40; + case ROCKSDB_NAMESPACE::Histograms:: + FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS: + return 0x41; case ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX: // 0x1F for backwards compatibility on current minor version. return 0x1F; @@ -5754,6 +5841,18 @@ class HistogramTypeJni { return ROCKSDB_NAMESPACE::Histograms::FILE_READ_COMPACTION_MICROS; case 0x3C: return ROCKSDB_NAMESPACE::Histograms::FILE_READ_DB_OPEN_MICROS; + case 0x3D: + return ROCKSDB_NAMESPACE::Histograms::FILE_READ_GET_MICROS; + case 0x3E: + return ROCKSDB_NAMESPACE::Histograms::FILE_READ_MULTIGET_MICROS; + case 0x3F: + return ROCKSDB_NAMESPACE::Histograms::FILE_READ_DB_ITERATOR_MICROS; + case 0x40: + return ROCKSDB_NAMESPACE::Histograms:: + FILE_READ_VERIFY_DB_CHECKSUM_MICROS; + case 0x41: + return ROCKSDB_NAMESPACE::Histograms:: + FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS; case 0x1F: // 0x1F for backwards compatibility on current minor version. return ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX; @@ -5886,6 +5985,52 @@ class MemoryUsageTypeJni { } }; +class PerfLevelTypeJni { + public: + static jbyte toJavaPerfLevelType(const ROCKSDB_NAMESPACE::PerfLevel level) { + switch (level) { + case ROCKSDB_NAMESPACE::PerfLevel::kUninitialized: + return 0x0; + case ROCKSDB_NAMESPACE::PerfLevel::kDisable: + return 0x1; + case ROCKSDB_NAMESPACE::PerfLevel::kEnableCount: + return 0x2; + case ROCKSDB_NAMESPACE::PerfLevel::kEnableTimeExceptForMutex: + return 0x3; + case ROCKSDB_NAMESPACE::PerfLevel::kEnableTimeAndCPUTimeExceptForMutex: + return 0x4; + case ROCKSDB_NAMESPACE::PerfLevel::kEnableTime: + return 0x5; + case ROCKSDB_NAMESPACE::PerfLevel::kOutOfBounds: + return 0x6; + default: + return 0x6; + } + } + + static ROCKSDB_NAMESPACE::PerfLevel toCppPerfLevelType(const jbyte level) { + switch (level) { + case 0x0: + return ROCKSDB_NAMESPACE::PerfLevel::kUninitialized; + case 0x1: + return ROCKSDB_NAMESPACE::PerfLevel::kDisable; + case 0x2: + return ROCKSDB_NAMESPACE::PerfLevel::kEnableCount; + case 0x3: + return ROCKSDB_NAMESPACE::PerfLevel::kEnableTimeExceptForMutex; + case 0x4: + return ROCKSDB_NAMESPACE::PerfLevel:: + kEnableTimeAndCPUTimeExceptForMutex; + case 0x5: + return ROCKSDB_NAMESPACE::PerfLevel::kEnableTime; + case 0x6: + return ROCKSDB_NAMESPACE::PerfLevel::kOutOfBounds; + default: + return ROCKSDB_NAMESPACE::PerfLevel::kOutOfBounds; + } + } +}; + // The portal class for org.rocksdb.Transaction class TransactionJni : public JavaClass { public: @@ -5899,7 +6044,7 @@ class TransactionJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/Transaction"); + return JavaClass::getJClass(env, "org/forstdb/Transaction"); } /** @@ -5927,7 +6072,7 @@ class TransactionJni : public JavaClass { jmethodID mid = env->GetMethodID( jclazz, "newWaitingTransactions", - "(JLjava/lang/String;[J)Lorg/rocksdb/Transaction$WaitingTransactions;"); + "(JLjava/lang/String;[J)Lorg/forstdb/Transaction$WaitingTransactions;"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError return nullptr; @@ -5988,7 +6133,7 @@ class TransactionDBJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TransactionDB"); + return JavaClass::getJClass(env, "org/forstdb/TransactionDB"); } /** @@ -6017,7 +6162,7 @@ class TransactionDBJni : public JavaClass { jmethodID mid = env->GetMethodID( jclazz, "newDeadlockInfo", - "(JJLjava/lang/String;Z)Lorg/rocksdb/TransactionDB$DeadlockInfo;"); + "(JJLjava/lang/String;Z)Lorg/forstdb/TransactionDB$DeadlockInfo;"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError return nullptr; @@ -6093,7 +6238,7 @@ class KeyLockInfoJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TransactionDB$KeyLockInfo"); + return JavaClass::getJClass(env, "org/forstdb/TransactionDB$KeyLockInfo"); } /** @@ -6164,7 +6309,7 @@ class DeadlockInfoJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TransactionDB$DeadlockInfo"); + return JavaClass::getJClass(env, "org/forstdb/TransactionDB$DeadlockInfo"); } }; @@ -6181,7 +6326,7 @@ class DeadlockPathJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TransactionDB$DeadlockPath"); + return JavaClass::getJClass(env, "org/forstdb/TransactionDB$DeadlockPath"); } /** @@ -6239,14 +6384,14 @@ class AbstractTableFilterJni } static jmethodID mid = - env->GetMethodID(jclazz, "filter", "(Lorg/rocksdb/TableProperties;)Z"); + env->GetMethodID(jclazz, "filter", "(Lorg/forstdb/TableProperties;)Z"); assert(mid != nullptr); return mid; } private: static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TableFilter"); + return JavaClass::getJClass(env, "org/forstdb/TableFilter"); } }; @@ -6422,7 +6567,7 @@ class TablePropertiesJni : public JavaClass { private: static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TableProperties"); + return JavaClass::getJClass(env, "org/forstdb/TableProperties"); } }; @@ -6438,7 +6583,7 @@ class ColumnFamilyDescriptorJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/ColumnFamilyDescriptor"); + return JavaClass::getJClass(env, "org/forstdb/ColumnFamilyDescriptor"); } /** @@ -6463,7 +6608,7 @@ class ColumnFamilyDescriptorJni : public JavaClass { } jmethodID mid = env->GetMethodID(jclazz, "", - "([BLorg/rocksdb/ColumnFamilyOptions;)V"); + "([BLorg/forstdb/ColumnFamilyOptions;)V"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError env->DeleteLocalRef(jcf_name); @@ -6515,7 +6660,7 @@ class ColumnFamilyDescriptorJni : public JavaClass { } static jmethodID mid = env->GetMethodID( - jclazz, "columnFamilyOptions", "()Lorg/rocksdb/ColumnFamilyOptions;"); + jclazz, "columnFamilyOptions", "()Lorg/forstdb/ColumnFamilyOptions;"); assert(mid != nullptr); return mid; } @@ -6648,7 +6793,7 @@ class ChecksumTypeJni { return ROCKSDB_NAMESPACE::ChecksumType::kXXH3; default: // undefined/default - return ROCKSDB_NAMESPACE::ChecksumType::kCRC32c; + return ROCKSDB_NAMESPACE::ChecksumType::kXXH3; } } }; @@ -6953,7 +7098,7 @@ class ThreadStatusJni : public JavaClass { * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/ThreadStatus"); + return JavaClass::getJClass(env, "org/forstdb/ThreadStatus"); } /** @@ -7277,7 +7422,7 @@ class LogFileJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/LogFile"); + return JavaClass::getJClass(env, "org/forstdb/LogFile"); } }; @@ -7302,7 +7447,7 @@ class LiveFileMetaDataJni : public JavaClass { jmethodID mid = env->GetMethodID( jclazz, "", - "([BILjava/lang/String;Ljava/lang/String;JJJ[B[BJZJJ)V"); + "([BILjava/lang/String;Ljava/lang/String;JJJ[B[BJZJJ[B)V"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError return nullptr; @@ -7353,6 +7498,18 @@ class LiveFileMetaDataJni : public JavaClass { return nullptr; } + jbyteArray jfile_checksum = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, live_file_meta_data->file_checksum); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + env->DeleteLocalRef(jsmallest_key); + env->DeleteLocalRef(jlargest_key); + return nullptr; + } + jobject jlive_file_meta_data = env->NewObject( jclazz, mid, jcolumn_family_name, static_cast(live_file_meta_data->level), jfile_name, jpath, @@ -7363,7 +7520,7 @@ class LiveFileMetaDataJni : public JavaClass { static_cast(live_file_meta_data->num_reads_sampled), static_cast(live_file_meta_data->being_compacted), static_cast(live_file_meta_data->num_entries), - static_cast(live_file_meta_data->num_deletions)); + static_cast(live_file_meta_data->num_deletions), jfile_checksum); if (env->ExceptionCheck()) { env->DeleteLocalRef(jcolumn_family_name); @@ -7371,6 +7528,7 @@ class LiveFileMetaDataJni : public JavaClass { env->DeleteLocalRef(jpath); env->DeleteLocalRef(jsmallest_key); env->DeleteLocalRef(jlargest_key); + env->DeleteLocalRef(jfile_checksum); return nullptr; } @@ -7380,12 +7538,13 @@ class LiveFileMetaDataJni : public JavaClass { env->DeleteLocalRef(jpath); env->DeleteLocalRef(jsmallest_key); env->DeleteLocalRef(jlargest_key); + env->DeleteLocalRef(jfile_checksum); return jlive_file_meta_data; } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/LiveFileMetaData"); + return JavaClass::getJClass(env, "org/forstdb/LiveFileMetaData"); } }; @@ -7410,7 +7569,8 @@ class SstFileMetaDataJni : public JavaClass { } jmethodID mid = env->GetMethodID( - jclazz, "", "(Ljava/lang/String;Ljava/lang/String;JJJ[B[BJZJJ)V"); + jclazz, "", + "(Ljava/lang/String;Ljava/lang/String;JJJ[B[BJZJJ[B)V"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError return nullptr; @@ -7450,6 +7610,17 @@ class SstFileMetaDataJni : public JavaClass { return nullptr; } + jbyteArray jfile_checksum = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, sst_file_meta_data->file_checksum); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + env->DeleteLocalRef(jsmallest_key); + env->DeleteLocalRef(jlargest_key); + return nullptr; + } + jobject jsst_file_meta_data = env->NewObject( jclazz, mid, jfile_name, jpath, static_cast(sst_file_meta_data->size), @@ -7458,13 +7629,14 @@ class SstFileMetaDataJni : public JavaClass { jlargest_key, static_cast(sst_file_meta_data->num_reads_sampled), static_cast(sst_file_meta_data->being_compacted), static_cast(sst_file_meta_data->num_entries), - static_cast(sst_file_meta_data->num_deletions)); + static_cast(sst_file_meta_data->num_deletions), jfile_checksum); if (env->ExceptionCheck()) { env->DeleteLocalRef(jfile_name); env->DeleteLocalRef(jpath); env->DeleteLocalRef(jsmallest_key); env->DeleteLocalRef(jlargest_key); + env->DeleteLocalRef(jfile_checksum); return nullptr; } @@ -7473,12 +7645,13 @@ class SstFileMetaDataJni : public JavaClass { env->DeleteLocalRef(jpath); env->DeleteLocalRef(jsmallest_key); env->DeleteLocalRef(jlargest_key); + env->DeleteLocalRef(jfile_checksum); return jsst_file_meta_data; } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/SstFileMetaData"); + return JavaClass::getJClass(env, "org/forstdb/SstFileMetaData"); } }; @@ -7502,7 +7675,7 @@ class LevelMetaDataJni : public JavaClass { } jmethodID mid = env->GetMethodID(jclazz, "", - "(IJ[Lorg/rocksdb/SstFileMetaData;)V"); + "(IJ[Lorg/forstdb/SstFileMetaData;)V"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError return nullptr; @@ -7544,7 +7717,7 @@ class LevelMetaDataJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/LevelMetaData"); + return JavaClass::getJClass(env, "org/forstdb/LevelMetaData"); } }; @@ -7647,7 +7820,7 @@ class ColumnFamilyMetaDataJni : public JavaClass { } jmethodID mid = env->GetMethodID(jclazz, "", - "(JJ[B[Lorg/rocksdb/LevelMetaData;)V"); + "(JJ[B[Lorg/forstdb/LevelMetaData;)V"); if (mid == nullptr) { // exception thrown: NoSuchMethodException or OutOfMemoryError return nullptr; @@ -7701,7 +7874,7 @@ class ColumnFamilyMetaDataJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/ColumnFamilyMetaData"); + return JavaClass::getJClass(env, "org/forstdb/ColumnFamilyMetaData"); } }; @@ -7722,7 +7895,7 @@ class AbstractTraceWriterJni */ static jclass getJClass(JNIEnv* env) { return RocksDBNativeClass::getJClass(env, - "org/rocksdb/AbstractTraceWriter"); + "org/forstdb/AbstractTraceWriter"); } /** @@ -7801,7 +7974,7 @@ class AbstractWalFilterJni * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown */ static jclass getJClass(JNIEnv* env) { - return RocksDBNativeClass::getJClass(env, "org/rocksdb/AbstractWalFilter"); + return RocksDBNativeClass::getJClass(env, "org/forstdb/AbstractWalFilter"); } /** @@ -8062,7 +8235,7 @@ class AbstractEventListenerJni */ static jclass getJClass(JNIEnv* env) { return RocksDBNativeClass::getJClass(env, - "org/rocksdb/AbstractEventListener"); + "org/forstdb/AbstractEventListener"); } /** @@ -8076,7 +8249,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onFlushCompletedProxy", - "(JLorg/rocksdb/FlushJobInfo;)V"); + "(JLorg/forstdb/FlushJobInfo;)V"); assert(mid != nullptr); return mid; } @@ -8092,7 +8265,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onFlushBeginProxy", - "(JLorg/rocksdb/FlushJobInfo;)V"); + "(JLorg/forstdb/FlushJobInfo;)V"); assert(mid != nullptr); return mid; } @@ -8108,7 +8281,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onTableFileDeleted", "(Lorg/rocksdb/TableFileDeletionInfo;)V"); + jclazz, "onTableFileDeleted", "(Lorg/forstdb/TableFileDeletionInfo;)V"); assert(mid != nullptr); return mid; } @@ -8125,7 +8298,7 @@ class AbstractEventListenerJni assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onCompactionBeginProxy", - "(JLorg/rocksdb/CompactionJobInfo;)V"); + "(JLorg/forstdb/CompactionJobInfo;)V"); assert(mid != nullptr); return mid; } @@ -8142,7 +8315,7 @@ class AbstractEventListenerJni assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onCompactionCompletedProxy", - "(JLorg/rocksdb/CompactionJobInfo;)V"); + "(JLorg/forstdb/CompactionJobInfo;)V"); assert(mid != nullptr); return mid; } @@ -8158,7 +8331,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onTableFileCreated", "(Lorg/rocksdb/TableFileCreationInfo;)V"); + jclazz, "onTableFileCreated", "(Lorg/forstdb/TableFileCreationInfo;)V"); assert(mid != nullptr); return mid; } @@ -8175,7 +8348,7 @@ class AbstractEventListenerJni assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onTableFileCreationStarted", - "(Lorg/rocksdb/TableFileCreationBriefInfo;)V"); + "(Lorg/forstdb/TableFileCreationBriefInfo;)V"); assert(mid != nullptr); return mid; } @@ -8191,7 +8364,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onMemTableSealed", - "(Lorg/rocksdb/MemTableInfo;)V"); + "(Lorg/forstdb/MemTableInfo;)V"); assert(mid != nullptr); return mid; } @@ -8209,7 +8382,7 @@ class AbstractEventListenerJni assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onColumnFamilyHandleDeletionStarted", - "(Lorg/rocksdb/ColumnFamilyHandle;)V"); + "(Lorg/forstdb/ColumnFamilyHandle;)V"); assert(mid != nullptr); return mid; } @@ -8226,7 +8399,7 @@ class AbstractEventListenerJni assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onExternalFileIngestedProxy", - "(JLorg/rocksdb/ExternalFileIngestionInfo;)V"); + "(JLorg/forstdb/ExternalFileIngestionInfo;)V"); assert(mid != nullptr); return mid; } @@ -8242,7 +8415,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onBackgroundErrorProxy", - "(BLorg/rocksdb/Status;)V"); + "(BLorg/forstdb/Status;)V"); assert(mid != nullptr); return mid; } @@ -8258,7 +8431,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onStallConditionsChanged", - "(Lorg/rocksdb/WriteStallInfo;)V"); + "(Lorg/forstdb/WriteStallInfo;)V"); assert(mid != nullptr); return mid; } @@ -8274,7 +8447,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onFileReadFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + jclazz, "onFileReadFinish", "(Lorg/forstdb/FileOperationInfo;)V"); assert(mid != nullptr); return mid; } @@ -8290,7 +8463,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onFileWriteFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + jclazz, "onFileWriteFinish", "(Lorg/forstdb/FileOperationInfo;)V"); assert(mid != nullptr); return mid; } @@ -8306,7 +8479,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onFileFlushFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + jclazz, "onFileFlushFinish", "(Lorg/forstdb/FileOperationInfo;)V"); assert(mid != nullptr); return mid; } @@ -8322,7 +8495,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onFileSyncFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + jclazz, "onFileSyncFinish", "(Lorg/forstdb/FileOperationInfo;)V"); assert(mid != nullptr); return mid; } @@ -8338,7 +8511,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onFileRangeSyncFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + jclazz, "onFileRangeSyncFinish", "(Lorg/forstdb/FileOperationInfo;)V"); assert(mid != nullptr); return mid; } @@ -8354,7 +8527,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onFileTruncateFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + jclazz, "onFileTruncateFinish", "(Lorg/forstdb/FileOperationInfo;)V"); assert(mid != nullptr); return mid; } @@ -8370,7 +8543,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID( - jclazz, "onFileCloseFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + jclazz, "onFileCloseFinish", "(Lorg/forstdb/FileOperationInfo;)V"); assert(mid != nullptr); return mid; } @@ -8402,7 +8575,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onErrorRecoveryBeginProxy", - "(BLorg/rocksdb/Status;)Z"); + "(BLorg/forstdb/Status;)Z"); assert(mid != nullptr); return mid; } @@ -8418,7 +8591,7 @@ class AbstractEventListenerJni jclass jclazz = getJClass(env); assert(jclazz != nullptr); static jmethodID mid = env->GetMethodID(jclazz, "onErrorRecoveryCompleted", - "(Lorg/rocksdb/Status;)V"); + "(Lorg/forstdb/Status;)V"); assert(mid != nullptr); return mid; } @@ -8472,7 +8645,7 @@ class FlushJobInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/FlushJobInfo"); + return JavaClass::getJClass(env, "org/forstdb/FlushJobInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { @@ -8518,13 +8691,13 @@ class TableFileDeletionInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TableFileDeletionInfo"); + return JavaClass::getJClass(env, "org/forstdb/TableFileDeletionInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { return env->GetMethodID( clazz, "", - "(Ljava/lang/String;Ljava/lang/String;ILorg/rocksdb/Status;)V"); + "(Ljava/lang/String;Ljava/lang/String;ILorg/forstdb/Status;)V"); } }; @@ -8542,7 +8715,7 @@ class CompactionJobInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/CompactionJobInfo"); + return JavaClass::getJClass(env, "org/forstdb/CompactionJobInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { @@ -8594,13 +8767,13 @@ class TableFileCreationInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TableFileCreationInfo"); + return JavaClass::getJClass(env, "org/forstdb/TableFileCreationInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { return env->GetMethodID( clazz, "", - "(JLorg/rocksdb/TableProperties;Lorg/rocksdb/Status;Ljava/lang/" + "(JLorg/forstdb/TableProperties;Lorg/forstdb/Status;Ljava/lang/" "String;Ljava/lang/String;Ljava/lang/String;IB)V"); } }; @@ -8634,7 +8807,7 @@ class TableFileCreationBriefInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/TableFileCreationBriefInfo"); + return JavaClass::getJClass(env, "org/forstdb/TableFileCreationBriefInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { @@ -8664,7 +8837,7 @@ class MemTableInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/MemTableInfo"); + return JavaClass::getJClass(env, "org/forstdb/MemTableInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { @@ -8711,13 +8884,13 @@ class ExternalFileIngestionInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/ExternalFileIngestionInfo"); + return JavaClass::getJClass(env, "org/forstdb/ExternalFileIngestionInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { return env->GetMethodID(clazz, "", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/" - "String;JLorg/rocksdb/TableProperties;)V"); + "String;JLorg/forstdb/TableProperties;)V"); } }; @@ -8739,7 +8912,7 @@ class WriteStallInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/WriteStallInfo"); + return JavaClass::getJClass(env, "org/forstdb/WriteStallInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { @@ -8772,12 +8945,12 @@ class FileOperationInfoJni : public JavaClass { } static jclass getJClass(JNIEnv* env) { - return JavaClass::getJClass(env, "org/rocksdb/FileOperationInfo"); + return JavaClass::getJClass(env, "org/forstdb/FileOperationInfo"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { return env->GetMethodID(clazz, "", - "(Ljava/lang/String;JJJJLorg/rocksdb/Status;)V"); + "(Ljava/lang/String;JJJJLorg/forstdb/Status;)V"); } }; @@ -8795,7 +8968,7 @@ class CompactRangeOptionsTimestampJni : public JavaClass { static jclass getJClass(JNIEnv* env) { return JavaClass::getJClass(env, - "org/rocksdb/CompactRangeOptions$Timestamp"); + "org/forstdb/CompactRangeOptions$Timestamp"); } static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { @@ -8803,5 +8976,101 @@ class CompactRangeOptionsTimestampJni : public JavaClass { } }; +// The portal class for org.rocksdb.BlockBasedTableOptions +class BlockBasedTableOptionsJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.BlockBasedTableConfig + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, + "org/forstdb/BlockBasedTableConfig"); + } + + /** + * Create a new Java org.rocksdb.BlockBasedTableConfig object with the + * properties as the provided C++ ROCKSDB_NAMESPACE::BlockBasedTableOptions + * object + * + * @param env A pointer to the Java environment + * @param cfoptions A pointer to ROCKSDB_NAMESPACE::ColumnFamilyOptions object + * + * @return A reference to a Java org.rocksdb.ColumnFamilyOptions object, or + * nullptr if an an exception occurs + */ + static jobject construct( + JNIEnv* env, const BlockBasedTableOptions* table_factory_options) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID method_id_init = + env->GetMethodID(jclazz, "", "(ZZZZBBDBZJIIIJZZZZZIIZZBBJD)V"); + if (method_id_init == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + FilterPolicyTypeJni filter_policy_type = + FilterPolicyTypeJni::kUnknownFilterPolicy; + jlong filter_policy_handle = 0L; + jdouble filter_policy_config_value = 0.0; + if (table_factory_options->filter_policy) { + auto filter_policy = table_factory_options->filter_policy.get(); + filter_policy_type = FilterPolicyJni::getFilterPolicyType( + filter_policy->CompatibilityName()); + if (FilterPolicyTypeJni::kUnknownFilterPolicy != filter_policy_type) { + filter_policy_handle = GET_CPLUSPLUS_POINTER(filter_policy); + } + } + + jobject jcfd = env->NewObject( + jclazz, method_id_init, + table_factory_options->cache_index_and_filter_blocks, + table_factory_options->cache_index_and_filter_blocks_with_high_priority, + table_factory_options->pin_l0_filter_and_index_blocks_in_cache, + table_factory_options->pin_top_level_index_and_filter, + IndexTypeJni::toJavaIndexType(table_factory_options->index_type), + DataBlockIndexTypeJni::toJavaDataBlockIndexType( + table_factory_options->data_block_index_type), + table_factory_options->data_block_hash_table_util_ratio, + ChecksumTypeJni::toJavaChecksumType(table_factory_options->checksum), + table_factory_options->no_block_cache, + static_cast(table_factory_options->block_size), + table_factory_options->block_size_deviation, + table_factory_options->block_restart_interval, + table_factory_options->index_block_restart_interval, + static_cast(table_factory_options->metadata_block_size), + table_factory_options->partition_filters, + table_factory_options->optimize_filters_for_memory, + table_factory_options->use_delta_encoding, + table_factory_options->whole_key_filtering, + table_factory_options->verify_compression, + table_factory_options->read_amp_bytes_per_bit, + table_factory_options->format_version, + table_factory_options->enable_index_compression, + table_factory_options->block_align, + IndexShorteningModeJni::toJavaIndexShorteningMode( + table_factory_options->index_shortening), + FilterPolicyJni::toJavaIndexType(filter_policy_type), + filter_policy_handle, filter_policy_config_value); + if (env->ExceptionCheck()) { + return nullptr; + } + + return jcfd; + } +}; + } // namespace ROCKSDB_NAMESPACE #endif // JAVA_ROCKSJNI_PORTAL_H_ diff --git a/java/rocksjni/ratelimiterjni.cc b/java/forstjni/ratelimiterjni.cc similarity index 79% rename from java/rocksjni/ratelimiterjni.cc rename to java/forstjni/ratelimiterjni.cc index 7a17f367e..83aead43c 100644 --- a/java/rocksjni/ratelimiterjni.cc +++ b/java/forstjni/ratelimiterjni.cc @@ -5,17 +5,17 @@ // // This file implements the "bridge" between Java and C++ for RateLimiter. -#include "include/org_rocksdb_RateLimiter.h" +#include "include/org_forstdb_RateLimiter.h" #include "rocksdb/rate_limiter.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_RateLimiter + * Class: org_forstdb_RateLimiter * Method: newRateLimiterHandle * Signature: (JJIBZ)J */ -jlong Java_org_rocksdb_RateLimiter_newRateLimiterHandle( +jlong Java_org_forstdb_RateLimiter_newRateLimiterHandle( JNIEnv* /*env*/, jclass /*jclazz*/, jlong jrate_bytes_per_second, jlong jrefill_period_micros, jint jfairness, jbyte jrate_limiter_mode, jboolean jauto_tune) { @@ -32,11 +32,11 @@ jlong Java_org_rocksdb_RateLimiter_newRateLimiterHandle( } /* - * Class: org_rocksdb_RateLimiter + * Class: org_forstdb_RateLimiter * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_RateLimiter_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_RateLimiter_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* handle = @@ -46,11 +46,11 @@ void Java_org_rocksdb_RateLimiter_disposeInternal(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_RateLimiter + * Class: org_forstdb_RateLimiter * Method: setBytesPerSecond * Signature: (JJ)V */ -void Java_org_rocksdb_RateLimiter_setBytesPerSecond(JNIEnv* /*env*/, +void Java_org_forstdb_RateLimiter_setBytesPerSecond(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jlong jbytes_per_second) { @@ -60,11 +60,11 @@ void Java_org_rocksdb_RateLimiter_setBytesPerSecond(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_RateLimiter + * Class: org_forstdb_RateLimiter * Method: getBytesPerSecond * Signature: (J)J */ -jlong Java_org_rocksdb_RateLimiter_getBytesPerSecond(JNIEnv* /*env*/, +jlong Java_org_forstdb_RateLimiter_getBytesPerSecond(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { return reinterpret_cast*>( @@ -74,11 +74,11 @@ jlong Java_org_rocksdb_RateLimiter_getBytesPerSecond(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_RateLimiter + * Class: org_forstdb_RateLimiter * Method: request * Signature: (JJ)V */ -void Java_org_rocksdb_RateLimiter_request(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_RateLimiter_request(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jlong jbytes) { reinterpret_cast*>(handle) ->get() @@ -86,11 +86,11 @@ void Java_org_rocksdb_RateLimiter_request(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_RateLimiter + * Class: org_forstdb_RateLimiter * Method: getSingleBurstBytes * Signature: (J)J */ -jlong Java_org_rocksdb_RateLimiter_getSingleBurstBytes(JNIEnv* /*env*/, +jlong Java_org_forstdb_RateLimiter_getSingleBurstBytes(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { return reinterpret_cast*>( @@ -100,11 +100,11 @@ jlong Java_org_rocksdb_RateLimiter_getSingleBurstBytes(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_RateLimiter + * Class: org_forstdb_RateLimiter * Method: getTotalBytesThrough * Signature: (J)J */ -jlong Java_org_rocksdb_RateLimiter_getTotalBytesThrough(JNIEnv* /*env*/, +jlong Java_org_forstdb_RateLimiter_getTotalBytesThrough(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { return reinterpret_cast*>( @@ -114,11 +114,11 @@ jlong Java_org_rocksdb_RateLimiter_getTotalBytesThrough(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_RateLimiter + * Class: org_forstdb_RateLimiter * Method: getTotalRequests * Signature: (J)J */ -jlong Java_org_rocksdb_RateLimiter_getTotalRequests(JNIEnv* /*env*/, +jlong Java_org_forstdb_RateLimiter_getTotalRequests(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { return reinterpret_cast*>( diff --git a/java/rocksjni/remove_emptyvalue_compactionfilterjni.cc b/java/forstjni/remove_emptyvalue_compactionfilterjni.cc similarity index 75% rename from java/rocksjni/remove_emptyvalue_compactionfilterjni.cc rename to java/forstjni/remove_emptyvalue_compactionfilterjni.cc index c0b09e151..2164fc44c 100644 --- a/java/rocksjni/remove_emptyvalue_compactionfilterjni.cc +++ b/java/forstjni/remove_emptyvalue_compactionfilterjni.cc @@ -5,16 +5,16 @@ #include -#include "include/org_rocksdb_RemoveEmptyValueCompactionFilter.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "include/org_forstdb_RemoveEmptyValueCompactionFilter.h" +#include "forstjni/cplusplus_to_java_convert.h" #include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h" /* - * Class: org_rocksdb_RemoveEmptyValueCompactionFilter + * Class: org_forstdb_RemoveEmptyValueCompactionFilter * Method: createNewRemoveEmptyValueCompactionFilter0 * Signature: ()J */ -jlong Java_org_rocksdb_RemoveEmptyValueCompactionFilter_createNewRemoveEmptyValueCompactionFilter0( +jlong Java_org_forstdb_RemoveEmptyValueCompactionFilter_createNewRemoveEmptyValueCompactionFilter0( JNIEnv* /*env*/, jclass /*jcls*/) { auto* compaction_filter = new ROCKSDB_NAMESPACE::RemoveEmptyValueCompactionFilter(); diff --git a/java/rocksjni/restorejni.cc b/java/forstjni/restorejni.cc similarity index 76% rename from java/rocksjni/restorejni.cc rename to java/forstjni/restorejni.cc index aadc86128..a20c883cc 100644 --- a/java/rocksjni/restorejni.cc +++ b/java/forstjni/restorejni.cc @@ -13,27 +13,27 @@ #include -#include "include/org_rocksdb_RestoreOptions.h" +#include "include/org_forstdb_RestoreOptions.h" #include "rocksdb/utilities/backup_engine.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_RestoreOptions + * Class: org_forstdb_RestoreOptions * Method: newRestoreOptions * Signature: (Z)J */ -jlong Java_org_rocksdb_RestoreOptions_newRestoreOptions( +jlong Java_org_forstdb_RestoreOptions_newRestoreOptions( JNIEnv* /*env*/, jclass /*jcls*/, jboolean keep_log_files) { auto* ropt = new ROCKSDB_NAMESPACE::RestoreOptions(keep_log_files); return GET_CPLUSPLUS_POINTER(ropt); } /* - * Class: org_rocksdb_RestoreOptions + * Class: org_forstdb_RestoreOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_RestoreOptions_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_RestoreOptions_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* ropt = reinterpret_cast(jhandle); diff --git a/java/rocksjni/rocks_callback_object.cc b/java/forstjni/rocks_callback_object.cc similarity index 82% rename from java/rocksjni/rocks_callback_object.cc rename to java/forstjni/rocks_callback_object.cc index 35513e151..19e8fe948 100644 --- a/java/rocksjni/rocks_callback_object.cc +++ b/java/forstjni/rocks_callback_object.cc @@ -4,19 +4,19 @@ // (found in the LICENSE.Apache file in the root directory). // // This file implements the "bridge" between Java and C++ for -// JNI Callbacks from C++ to sub-classes or org.rocksdb.RocksCallbackObject +// JNI Callbacks from C++ to sub-classes or org.forstdb.RocksCallbackObject #include -#include "include/org_rocksdb_RocksCallbackObject.h" +#include "include/org_forstdb_RocksCallbackObject.h" #include "jnicallback.h" /* - * Class: org_rocksdb_RocksCallbackObject + * Class: org_forstdb_RocksCallbackObject * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_RocksCallbackObject_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_RocksCallbackObject_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { // TODO(AR) is deleting from the super class JniCallback OK, or must we delete diff --git a/java/rocksjni/rocksdb_exception_test.cc b/java/forstjni/rocksdb_exception_test.cc similarity index 72% rename from java/rocksjni/rocksdb_exception_test.cc rename to java/forstjni/rocksdb_exception_test.cc index 67e62f726..8150bb1ad 100644 --- a/java/rocksjni/rocksdb_exception_test.cc +++ b/java/forstjni/rocksdb_exception_test.cc @@ -5,50 +5,50 @@ #include -#include "include/org_rocksdb_RocksDBExceptionTest.h" +#include "include/org_forstdb_RocksDBExceptionTest.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_RocksDBExceptionTest + * Class: org_forstdb_RocksDBExceptionTest * Method: raiseException * Signature: ()V */ -void Java_org_rocksdb_RocksDBExceptionTest_raiseException(JNIEnv* env, +void Java_org_forstdb_RocksDBExceptionTest_raiseException(JNIEnv* env, jobject /*jobj*/) { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, std::string("test message")); } /* - * Class: org_rocksdb_RocksDBExceptionTest + * Class: org_forstdb_RocksDBExceptionTest * Method: raiseExceptionWithStatusCode * Signature: ()V */ -void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionWithStatusCode( +void Java_org_forstdb_RocksDBExceptionTest_raiseExceptionWithStatusCode( JNIEnv* env, jobject /*jobj*/) { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( env, "test message", ROCKSDB_NAMESPACE::Status::NotSupported()); } /* - * Class: org_rocksdb_RocksDBExceptionTest + * Class: org_forstdb_RocksDBExceptionTest * Method: raiseExceptionNoMsgWithStatusCode * Signature: ()V */ -void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCode( +void Java_org_forstdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCode( JNIEnv* env, jobject /*jobj*/) { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( env, ROCKSDB_NAMESPACE::Status::NotSupported()); } /* - * Class: org_rocksdb_RocksDBExceptionTest + * Class: org_forstdb_RocksDBExceptionTest * Method: raiseExceptionWithStatusCodeSubCode * Signature: ()V */ -void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeSubCode( +void Java_org_forstdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeSubCode( JNIEnv* env, jobject /*jobj*/) { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( env, "test message", @@ -57,11 +57,11 @@ void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeSubCode( } /* - * Class: org_rocksdb_RocksDBExceptionTest + * Class: org_forstdb_RocksDBExceptionTest * Method: raiseExceptionNoMsgWithStatusCodeSubCode * Signature: ()V */ -void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCodeSubCode( +void Java_org_forstdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCodeSubCode( JNIEnv* env, jobject /*jobj*/) { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( env, ROCKSDB_NAMESPACE::Status::TimedOut( @@ -69,11 +69,11 @@ void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCodeSubC } /* - * Class: org_rocksdb_RocksDBExceptionTest + * Class: org_forstdb_RocksDBExceptionTest * Method: raiseExceptionWithStatusCodeState * Signature: ()V */ -void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeState( +void Java_org_forstdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeState( JNIEnv* env, jobject /*jobj*/) { ROCKSDB_NAMESPACE::Slice state("test state"); ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( diff --git a/java/rocksjni/rocksjni.cc b/java/forstjni/rocksjni.cc similarity index 85% rename from java/rocksjni/rocksjni.cc rename to java/forstjni/rocksjni.cc index 8c2b999e0..52767ece4 100644 --- a/java/rocksjni/rocksjni.cc +++ b/java/forstjni/rocksjni.cc @@ -17,15 +17,17 @@ #include #include -#include "include/org_rocksdb_RocksDB.h" +#include "include/org_forstdb_RocksDB.h" #include "rocksdb/cache.h" #include "rocksdb/convenience.h" #include "rocksdb/db.h" #include "rocksdb/options.h" +#include "rocksdb/perf_context.h" #include "rocksdb/types.h" #include "rocksdb/version.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/kv_helper.h" +#include "forstjni/portal.h" #ifdef min #undef min @@ -57,11 +59,11 @@ jlong rocksdb_open_helper(JNIEnv* env, jlong jopt_handle, jstring jdb_path, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: open * Signature: (JLjava/lang/String;)J */ -jlong Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2(JNIEnv* env, jclass, +jlong Java_org_forstdb_RocksDB_open__JLjava_lang_String_2(JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path) { return rocksdb_open_helper(env, jopt_handle, jdb_path, @@ -72,11 +74,11 @@ jlong Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: openROnly * Signature: (JLjava/lang/String;Z)J */ -jlong Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2Z( +jlong Java_org_forstdb_RocksDB_openROnly__JLjava_lang_String_2Z( JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, jboolean jerror_if_wal_file_exists) { const bool error_if_wal_file_exists = jerror_if_wal_file_exists == JNI_TRUE; @@ -176,11 +178,11 @@ jlongArray rocksdb_open_helper( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: openROnly * Signature: (JLjava/lang/String;[[B[JZ)[J */ -jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3JZ( +jlongArray Java_org_forstdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3JZ( JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, jobjectArray jcolumn_names, jlongArray jcolumn_options, jboolean jerror_if_wal_file_exists) { @@ -201,11 +203,11 @@ jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3JZ( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: open * Signature: (JLjava/lang/String;[[B[J)[J */ -jlongArray Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2_3_3B_3J( +jlongArray Java_org_forstdb_RocksDB_open__JLjava_lang_String_2_3_3B_3J( JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, jobjectArray jcolumn_names, jlongArray jcolumn_options) { return rocksdb_open_helper( @@ -219,11 +221,11 @@ jlongArray Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2_3_3B_3J( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: openAsSecondary * Signature: (JLjava/lang/String;Ljava/lang/String;)J */ -jlong Java_org_rocksdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_2( +jlong Java_org_forstdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_2( JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, jstring jsecondary_db_path) { const char* secondary_db_path = @@ -249,12 +251,12 @@ jlong Java_org_rocksdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_S } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: openAsSecondary * Signature: (JLjava/lang/String;Ljava/lang/String;[[B[J)[J */ jlongArray -Java_org_rocksdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_2_3_3B_3J( +Java_org_forstdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_2_3_3B_3J( JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, jstring jsecondary_db_path, jobjectArray jcolumn_names, jlongArray jcolumn_options) { @@ -285,22 +287,22 @@ Java_org_rocksdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_ } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_disposeInternal(JNIEnv*, jobject, jlong jhandle) { +void Java_org_forstdb_RocksDB_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* db = reinterpret_cast(jhandle); assert(db != nullptr); delete db; } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: closeDatabase * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_closeDatabase(JNIEnv* env, jclass, +void Java_org_forstdb_RocksDB_closeDatabase(JNIEnv* env, jclass, jlong jhandle) { auto* db = reinterpret_cast(jhandle); assert(db != nullptr); @@ -309,11 +311,11 @@ void Java_org_rocksdb_RocksDB_closeDatabase(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: listColumnFamilies * Signature: (JLjava/lang/String;)[[B */ -jobjectArray Java_org_rocksdb_RocksDB_listColumnFamilies(JNIEnv* env, jclass, +jobjectArray Java_org_forstdb_RocksDB_listColumnFamilies(JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path) { std::vector column_family_names; @@ -336,11 +338,11 @@ jobjectArray Java_org_rocksdb_RocksDB_listColumnFamilies(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: createColumnFamily * Signature: (J[BIJ)J */ -jlong Java_org_rocksdb_RocksDB_createColumnFamily(JNIEnv* env, jobject, +jlong Java_org_forstdb_RocksDB_createColumnFamily(JNIEnv* env, jobject, jlong jhandle, jbyteArray jcf_name, jint jcf_name_len, @@ -372,11 +374,11 @@ jlong Java_org_rocksdb_RocksDB_createColumnFamily(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: createColumnFamilies * Signature: (JJ[[B)[J */ -jlongArray Java_org_rocksdb_RocksDB_createColumnFamilies__JJ_3_3B( +jlongArray Java_org_forstdb_RocksDB_createColumnFamilies__JJ_3_3B( JNIEnv* env, jobject, jlong jhandle, jlong jcf_options_handle, jobjectArray jcf_names) { auto* db = reinterpret_cast(jhandle); @@ -413,11 +415,11 @@ jlongArray Java_org_rocksdb_RocksDB_createColumnFamilies__JJ_3_3B( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: createColumnFamilies * Signature: (J[J[[B)[J */ -jlongArray Java_org_rocksdb_RocksDB_createColumnFamilies__J_3J_3_3B( +jlongArray Java_org_forstdb_RocksDB_createColumnFamilies__J_3J_3_3B( JNIEnv* env, jobject, jlong jhandle, jlongArray jcf_options_handles, jobjectArray jcf_names) { auto* db = reinterpret_cast(jhandle); @@ -490,11 +492,11 @@ jlongArray Java_org_rocksdb_RocksDB_createColumnFamilies__J_3J_3_3B( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: createColumnFamilyWithImport * Signature: (J[BIJJ[J)J */ -jlong Java_org_rocksdb_RocksDB_createColumnFamilyWithImport( +jlong Java_org_forstdb_RocksDB_createColumnFamilyWithImport( JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jcf_name, jint jcf_name_len, jlong j_cf_options, jlong j_cf_import_options, jlongArray j_metadata_handle_array) { @@ -547,11 +549,11 @@ jlong Java_org_rocksdb_RocksDB_createColumnFamilyWithImport( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: dropColumnFamily * Signature: (JJ)V; */ -void Java_org_rocksdb_RocksDB_dropColumnFamily(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_dropColumnFamily(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle) { auto* db_handle = reinterpret_cast(jdb_handle); @@ -564,11 +566,11 @@ void Java_org_rocksdb_RocksDB_dropColumnFamily(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: dropColumnFamilies * Signature: (J[J)V */ -void Java_org_rocksdb_RocksDB_dropColumnFamilies( +void Java_org_forstdb_RocksDB_dropColumnFamilies( JNIEnv* env, jobject, jlong jdb_handle, jlongArray jcolumn_family_handles) { auto* db_handle = reinterpret_cast(jdb_handle); @@ -599,61 +601,12 @@ void Java_org_rocksdb_RocksDB_dropColumnFamilies( ////////////////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::DB::Put -/** - * @return true if the put succeeded, false if a Java Exception was thrown - */ -bool rocksdb_put_helper(JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, - const ROCKSDB_NAMESPACE::WriteOptions& write_options, - ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle, - jbyteArray jkey, jint jkey_off, jint jkey_len, - jbyteArray jval, jint jval_off, jint jval_len) { - jbyte* key = new jbyte[jkey_len]; - env->GetByteArrayRegion(jkey, jkey_off, jkey_len, key); - if (env->ExceptionCheck()) { - // exception thrown: ArrayIndexOutOfBoundsException - delete[] key; - return false; - } - - jbyte* value = new jbyte[jval_len]; - env->GetByteArrayRegion(jval, jval_off, jval_len, value); - if (env->ExceptionCheck()) { - // exception thrown: ArrayIndexOutOfBoundsException - delete[] value; - delete[] key; - return false; - } - - ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); - ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), - jval_len); - - ROCKSDB_NAMESPACE::Status s; - if (cf_handle != nullptr) { - s = db->Put(write_options, cf_handle, key_slice, value_slice); - } else { - // backwards compatibility - s = db->Put(write_options, key_slice, value_slice); - } - - // cleanup - delete[] value; - delete[] key; - - if (s.ok()) { - return true; - } else { - ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); - return false; - } -} - /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: put * Signature: (J[BII[BII)V */ -void Java_org_rocksdb_RocksDB_put__J_3BII_3BII(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_put__J_3BII_3BII(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, @@ -661,16 +614,22 @@ void Java_org_rocksdb_RocksDB_put__J_3BII_3BII(JNIEnv* env, jobject, auto* db = reinterpret_cast(jdb_handle); static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = ROCKSDB_NAMESPACE::WriteOptions(); - rocksdb_put_helper(env, db, default_write_options, nullptr, jkey, jkey_off, - jkey_len, jval, jval_off, jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, db->Put(default_write_options, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: put * Signature: (J[BII[BIIJ)V */ -void Java_org_rocksdb_RocksDB_put__J_3BII_3BIIJ(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_put__J_3BII_3BIIJ(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, @@ -681,22 +640,30 @@ void Java_org_rocksdb_RocksDB_put__J_3BII_3BIIJ(JNIEnv* env, jobject, ROCKSDB_NAMESPACE::WriteOptions(); auto* cf_handle = reinterpret_cast(jcf_handle); - if (cf_handle != nullptr) { - rocksdb_put_helper(env, db, default_write_options, cf_handle, jkey, - jkey_off, jkey_len, jval, jval_off, jval_len); - } else { + if (cf_handle == nullptr) { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( env, ROCKSDB_NAMESPACE::Status::InvalidArgument( "Invalid ColumnFamilyHandle.")); + return; + } + + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, + db->Put(default_write_options, cf_handle, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; } } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: put * Signature: (JJ[BII[BII)V */ -void Java_org_rocksdb_RocksDB_put__JJ_3BII_3BII(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_put__JJ_3BII_3BII(JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, jbyteArray jkey, jint jkey_off, @@ -705,16 +672,23 @@ void Java_org_rocksdb_RocksDB_put__JJ_3BII_3BII(JNIEnv* env, jobject, auto* db = reinterpret_cast(jdb_handle); auto* write_options = reinterpret_cast(jwrite_options_handle); - rocksdb_put_helper(env, db, *write_options, nullptr, jkey, jkey_off, jkey_len, - jval, jval_off, jval_len); + + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, db->Put(*write_options, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: put * Signature: (JJ[BII[BIIJ)V */ -void Java_org_rocksdb_RocksDB_put__JJ_3BII_3BIIJ( +void Java_org_forstdb_RocksDB_put__JJ_3BII_3BIIJ( JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, jint jval_off, jint jval_len, jlong jcf_handle) { @@ -723,22 +697,28 @@ void Java_org_rocksdb_RocksDB_put__JJ_3BII_3BIIJ( reinterpret_cast(jwrite_options_handle); auto* cf_handle = reinterpret_cast(jcf_handle); - if (cf_handle != nullptr) { - rocksdb_put_helper(env, db, *write_options, cf_handle, jkey, jkey_off, - jkey_len, jval, jval_off, jval_len); - } else { + if (cf_handle == nullptr) { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( env, ROCKSDB_NAMESPACE::Status::InvalidArgument( "Invalid ColumnFamilyHandle.")); + return; + } + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, db->Put(*write_options, cf_handle, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; } } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: putDirect * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V */ -void Java_org_rocksdb_RocksDB_putDirect( +void Java_org_forstdb_RocksDB_putDirect( JNIEnv* env, jobject /*jdb*/, jlong jdb_handle, jlong jwrite_options_handle, jobject jkey, jint jkey_off, jint jkey_len, jobject jval, jint jval_off, jint jval_len, jlong jcf_handle) { @@ -804,11 +784,11 @@ bool rocksdb_delete_helper(JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: delete * Signature: (J[BII)V */ -void Java_org_rocksdb_RocksDB_delete__J_3BII(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_delete__J_3BII(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len) { auto* db = reinterpret_cast(jdb_handle); @@ -819,11 +799,11 @@ void Java_org_rocksdb_RocksDB_delete__J_3BII(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: delete * Signature: (J[BIIJ)V */ -void Java_org_rocksdb_RocksDB_delete__J_3BIIJ(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_delete__J_3BIIJ(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jlong jcf_handle) { @@ -843,11 +823,11 @@ void Java_org_rocksdb_RocksDB_delete__J_3BIIJ(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: delete * Signature: (JJ[BII)V */ -void Java_org_rocksdb_RocksDB_delete__JJ_3BII(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_delete__JJ_3BII(JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, jbyteArray jkey, jint jkey_off, @@ -860,11 +840,11 @@ void Java_org_rocksdb_RocksDB_delete__JJ_3BII(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: delete * Signature: (JJ[BIIJ)V */ -void Java_org_rocksdb_RocksDB_delete__JJ_3BIIJ( +void Java_org_forstdb_RocksDB_delete__JJ_3BIIJ( JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, jbyteArray jkey, jint jkey_off, jint jkey_len, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -921,11 +901,11 @@ bool rocksdb_single_delete_helper( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: singleDelete * Signature: (J[BI)V */ -void Java_org_rocksdb_RocksDB_singleDelete__J_3BI(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_singleDelete__J_3BI(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_len) { @@ -937,11 +917,11 @@ void Java_org_rocksdb_RocksDB_singleDelete__J_3BI(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: singleDelete * Signature: (J[BIJ)V */ -void Java_org_rocksdb_RocksDB_singleDelete__J_3BIJ(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_singleDelete__J_3BIJ(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_len, @@ -962,11 +942,11 @@ void Java_org_rocksdb_RocksDB_singleDelete__J_3BIJ(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: singleDelete * Signature: (JJ[BIJ)V */ -void Java_org_rocksdb_RocksDB_singleDelete__JJ_3BI(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_singleDelete__JJ_3BI(JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, jbyteArray jkey, @@ -979,11 +959,11 @@ void Java_org_rocksdb_RocksDB_singleDelete__JJ_3BI(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: singleDelete * Signature: (JJ[BIJ)V */ -void Java_org_rocksdb_RocksDB_singleDelete__JJ_3BIJ( +void Java_org_forstdb_RocksDB_singleDelete__JJ_3BIJ( JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, jbyteArray jkey, jint jkey_len, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -1035,8 +1015,13 @@ bool rocksdb_delete_range_helper( ROCKSDB_NAMESPACE::Slice end_key_slice(reinterpret_cast(end_key), jend_key_len); - ROCKSDB_NAMESPACE::Status s = - db->DeleteRange(write_options, cf_handle, begin_key_slice, end_key_slice); + ROCKSDB_NAMESPACE::Status s; + if (cf_handle != nullptr) { + s = db->DeleteRange(write_options, cf_handle, begin_key_slice, + end_key_slice); + } else { + s = db->DeleteRange(write_options, begin_key_slice, end_key_slice); + } // cleanup delete[] begin_key; @@ -1051,11 +1036,11 @@ bool rocksdb_delete_range_helper( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: deleteRange * Signature: (J[BII[BII)V */ -void Java_org_rocksdb_RocksDB_deleteRange__J_3BII_3BII( +void Java_org_forstdb_RocksDB_deleteRange__J_3BII_3BII( JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jbegin_key, jint jbegin_key_off, jint jbegin_key_len, jbyteArray jend_key, jint jend_key_off, jint jend_key_len) { @@ -1155,11 +1140,11 @@ jint rocksdb_get_helper_direct( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: deleteRange * Signature: (J[BII[BIIJ)V */ -void Java_org_rocksdb_RocksDB_deleteRange__J_3BII_3BIIJ( +void Java_org_forstdb_RocksDB_deleteRange__J_3BII_3BIIJ( JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jbegin_key, jint jbegin_key_off, jint jbegin_key_len, jbyteArray jend_key, jint jend_key_off, jint jend_key_len, jlong jcf_handle) { @@ -1180,11 +1165,11 @@ void Java_org_rocksdb_RocksDB_deleteRange__J_3BII_3BIIJ( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: deleteRange * Signature: (JJ[BII[BII)V */ -void Java_org_rocksdb_RocksDB_deleteRange__JJ_3BII_3BII( +void Java_org_forstdb_RocksDB_deleteRange__JJ_3BII_3BII( JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, jbyteArray jbegin_key, jint jbegin_key_off, jint jbegin_key_len, jbyteArray jend_key, jint jend_key_off, jint jend_key_len) { @@ -1197,11 +1182,11 @@ void Java_org_rocksdb_RocksDB_deleteRange__JJ_3BII_3BII( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: deleteRange * Signature: (JJ[BII[BIIJ)V */ -void Java_org_rocksdb_RocksDB_deleteRange__JJ_3BII_3BIIJ( +void Java_org_forstdb_RocksDB_deleteRange__JJ_3BII_3BIIJ( JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, jbyteArray jbegin_key, jint jbegin_key_off, jint jbegin_key_len, jbyteArray jend_key, jint jend_key_off, jint jend_key_len, @@ -1223,11 +1208,66 @@ void Java_org_rocksdb_RocksDB_deleteRange__JJ_3BII_3BIIJ( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB + * Method: clipColumnFamily + * Signature: (JJ[BII[BII)V + */ +void Java_org_forstdb_RocksDB_clipColumnFamily( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jbyteArray jbegin_key, jint jbegin_key_off, jint jbegin_key_len, + jbyteArray jend_key, jint jend_key_off, jint jend_key_len) { + auto* db = reinterpret_cast(jdb_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + jbyte* begin_key = new jbyte[jbegin_key_len]; + env->GetByteArrayRegion(jbegin_key, jbegin_key_off, jbegin_key_len, + begin_key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] begin_key; + return; + } + ROCKSDB_NAMESPACE::Slice begin_key_slice(reinterpret_cast(begin_key), + jbegin_key_len); + + jbyte* end_key = new jbyte[jend_key_len]; + env->GetByteArrayRegion(jend_key, jend_key_off, jend_key_len, end_key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] begin_key; + delete[] end_key; + return; + } + ROCKSDB_NAMESPACE::Slice end_key_slice(reinterpret_cast(end_key), + jend_key_len); + + ROCKSDB_NAMESPACE::Status s = + db->ClipColumnFamily(cf_handle, begin_key_slice, end_key_slice); + + // cleanup + delete[] begin_key; + delete[] end_key; + + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return; + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +/* + * Class: org_forstdb_RocksDB * Method: getDirect * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)I */ -jint Java_org_rocksdb_RocksDB_getDirect(JNIEnv* env, jobject /*jdb*/, +jint Java_org_forstdb_RocksDB_getDirect(JNIEnv* env, jobject /*jdb*/, jlong jdb_handle, jlong jropt_handle, jobject jkey, jint jkey_off, jint jkey_len, jobject jval, @@ -1248,59 +1288,12 @@ jint Java_org_rocksdb_RocksDB_getDirect(JNIEnv* env, jobject /*jdb*/, ////////////////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::DB::Merge -/** - * @return true if the merge succeeded, false if a Java Exception was thrown - */ -bool rocksdb_merge_helper(JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, - const ROCKSDB_NAMESPACE::WriteOptions& write_options, - ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle, - jbyteArray jkey, jint jkey_off, jint jkey_len, - jbyteArray jval, jint jval_off, jint jval_len) { - jbyte* key = new jbyte[jkey_len]; - env->GetByteArrayRegion(jkey, jkey_off, jkey_len, key); - if (env->ExceptionCheck()) { - // exception thrown: ArrayIndexOutOfBoundsException - delete[] key; - return false; - } - ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); - - jbyte* value = new jbyte[jval_len]; - env->GetByteArrayRegion(jval, jval_off, jval_len, value); - if (env->ExceptionCheck()) { - // exception thrown: ArrayIndexOutOfBoundsException - delete[] value; - delete[] key; - return false; - } - ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), - jval_len); - - ROCKSDB_NAMESPACE::Status s; - if (cf_handle != nullptr) { - s = db->Merge(write_options, cf_handle, key_slice, value_slice); - } else { - s = db->Merge(write_options, key_slice, value_slice); - } - - // cleanup - delete[] value; - delete[] key; - - if (s.ok()) { - return true; - } - - ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); - return false; -} - /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: merge * Signature: (J[BII[BII)V */ -void Java_org_rocksdb_RocksDB_merge__J_3BII_3BII(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_merge__J_3BII_3BII(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, @@ -1308,16 +1301,22 @@ void Java_org_rocksdb_RocksDB_merge__J_3BII_3BII(JNIEnv* env, jobject, auto* db = reinterpret_cast(jdb_handle); static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = ROCKSDB_NAMESPACE::WriteOptions(); - rocksdb_merge_helper(env, db, default_write_options, nullptr, jkey, jkey_off, - jkey_len, jval, jval_off, jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, db->Merge(default_write_options, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: merge * Signature: (J[BII[BIIJ)V */ -void Java_org_rocksdb_RocksDB_merge__J_3BII_3BIIJ( +void Java_org_forstdb_RocksDB_merge__J_3BII_3BIIJ( JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, jint jval_off, jint jval_len, jlong jcf_handle) { @@ -1327,8 +1326,15 @@ void Java_org_rocksdb_RocksDB_merge__J_3BII_3BIIJ( auto* cf_handle = reinterpret_cast(jcf_handle); if (cf_handle != nullptr) { - rocksdb_merge_helper(env, db, default_write_options, cf_handle, jkey, - jkey_off, jkey_len, jval, jval_off, jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, db->Merge(default_write_options, cf_handle, key.slice(), + value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } else { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( env, ROCKSDB_NAMESPACE::Status::InvalidArgument( @@ -1337,27 +1343,33 @@ void Java_org_rocksdb_RocksDB_merge__J_3BII_3BIIJ( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: merge * Signature: (JJ[BII[BII)V */ -void Java_org_rocksdb_RocksDB_merge__JJ_3BII_3BII( +void Java_org_forstdb_RocksDB_merge__JJ_3BII_3BII( JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, jint jval_off, jint jval_len) { auto* db = reinterpret_cast(jdb_handle); auto* write_options = reinterpret_cast(jwrite_options_handle); - rocksdb_merge_helper(env, db, *write_options, nullptr, jkey, jkey_off, - jkey_len, jval, jval_off, jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, db->Merge(*write_options, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: merge * Signature: (JJ[BII[BIIJ)V */ -void Java_org_rocksdb_RocksDB_merge__JJ_3BII_3BIIJ( +void Java_org_forstdb_RocksDB_merge__JJ_3BII_3BIIJ( JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, jint jval_off, jint jval_len, jlong jcf_handle) { @@ -1367,8 +1379,15 @@ void Java_org_rocksdb_RocksDB_merge__JJ_3BII_3BIIJ( auto* cf_handle = reinterpret_cast(jcf_handle); if (cf_handle != nullptr) { - rocksdb_merge_helper(env, db, *write_options, cf_handle, jkey, jkey_off, - jkey_len, jval, jval_off, jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, + db->Merge(*write_options, cf_handle, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } else { ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( env, ROCKSDB_NAMESPACE::Status::InvalidArgument( @@ -1376,24 +1395,45 @@ void Java_org_rocksdb_RocksDB_merge__JJ_3BII_3BIIJ( } } -jlong rocksdb_iterator_helper( - ROCKSDB_NAMESPACE::DB* db, ROCKSDB_NAMESPACE::ReadOptions read_options, - ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle) { - ROCKSDB_NAMESPACE::Iterator* iterator = nullptr; - if (cf_handle != nullptr) { - iterator = db->NewIterator(read_options, cf_handle); - } else { - iterator = db->NewIterator(read_options); - } - return GET_CPLUSPLUS_POINTER(iterator); +/* + * Class: org_forstdb_RocksDB + * Method: mergeDirect + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +void Java_org_forstdb_RocksDB_mergeDirect( + JNIEnv* env, jobject /*jdb*/, jlong jdb_handle, jlong jwrite_options_handle, + jobject jkey, jint jkey_off, jint jkey_len, jobject jval, jint jval_off, + jint jval_len, jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + + auto merge = [&env, &db, &cf_handle, &write_options]( + ROCKSDB_NAMESPACE::Slice& key, + ROCKSDB_NAMESPACE::Slice& value) { + ROCKSDB_NAMESPACE::Status s; + if (cf_handle == nullptr) { + s = db->Merge(*write_options, key, value); + } else { + s = db->Merge(*write_options, cf_handle, key, value); + } + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + }; + ROCKSDB_NAMESPACE::JniUtil::kv_op_direct(merge, env, jkey, jkey_off, jkey_len, + jval, jval_off, jval_len); } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: deleteDirect * Signature: (JJLjava/nio/ByteBuffer;IIJ)V */ -void Java_org_rocksdb_RocksDB_deleteDirect(JNIEnv* env, jobject /*jdb*/, +void Java_org_forstdb_RocksDB_deleteDirect(JNIEnv* env, jobject /*jdb*/, jlong jdb_handle, jlong jwrite_options, jobject jkey, jint jkey_offset, jint jkey_len, @@ -1423,11 +1463,11 @@ void Java_org_rocksdb_RocksDB_deleteDirect(JNIEnv* env, jobject /*jdb*/, ////////////////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::DB::Write /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: write0 * Signature: (JJJ)V */ -void Java_org_rocksdb_RocksDB_write0(JNIEnv* env, jobject, jlong jdb_handle, +void Java_org_forstdb_RocksDB_write0(JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, jlong jwb_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -1443,11 +1483,11 @@ void Java_org_rocksdb_RocksDB_write0(JNIEnv* env, jobject, jlong jdb_handle, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: write1 * Signature: (JJJ)V */ -void Java_org_rocksdb_RocksDB_write1(JNIEnv* env, jobject, jlong jdb_handle, +void Java_org_forstdb_RocksDB_write1(JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, jlong jwbwi_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -1514,11 +1554,11 @@ jbyteArray rocksdb_get_helper( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: get * Signature: (J[BII)[B */ -jbyteArray Java_org_rocksdb_RocksDB_get__J_3BII(JNIEnv* env, jobject, +jbyteArray Java_org_forstdb_RocksDB_get__J_3BII(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len) { @@ -1528,11 +1568,11 @@ jbyteArray Java_org_rocksdb_RocksDB_get__J_3BII(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: get * Signature: (J[BIIJ)[B */ -jbyteArray Java_org_rocksdb_RocksDB_get__J_3BIIJ(JNIEnv* env, jobject, +jbyteArray Java_org_forstdb_RocksDB_get__J_3BIIJ(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, @@ -1552,11 +1592,11 @@ jbyteArray Java_org_rocksdb_RocksDB_get__J_3BIIJ(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: get * Signature: (JJ[BII)[B */ -jbyteArray Java_org_rocksdb_RocksDB_get__JJ_3BII(JNIEnv* env, jobject, +jbyteArray Java_org_forstdb_RocksDB_get__JJ_3BII(JNIEnv* env, jobject, jlong jdb_handle, jlong jropt_handle, jbyteArray jkey, jint jkey_off, @@ -1568,11 +1608,11 @@ jbyteArray Java_org_rocksdb_RocksDB_get__JJ_3BII(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: get * Signature: (JJ[BIIJ)[B */ -jbyteArray Java_org_rocksdb_RocksDB_get__JJ_3BIIJ( +jbyteArray Java_org_forstdb_RocksDB_get__JJ_3BIIJ( JNIEnv* env, jobject, jlong jdb_handle, jlong jropt_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jlong jcf_handle) { auto* db_handle = reinterpret_cast(jdb_handle); @@ -1657,11 +1697,11 @@ jint rocksdb_get_helper( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: get * Signature: (J[BII[BII)I */ -jint Java_org_rocksdb_RocksDB_get__J_3BII_3BII(JNIEnv* env, jobject, +jint Java_org_forstdb_RocksDB_get__J_3BII_3BII(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, @@ -1674,11 +1714,11 @@ jint Java_org_rocksdb_RocksDB_get__J_3BII_3BII(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: get * Signature: (J[BII[BIIJ)I */ -jint Java_org_rocksdb_RocksDB_get__J_3BII_3BIIJ(JNIEnv* env, jobject, +jint Java_org_forstdb_RocksDB_get__J_3BII_3BIIJ(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, @@ -1702,11 +1742,11 @@ jint Java_org_rocksdb_RocksDB_get__J_3BII_3BIIJ(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: get * Signature: (JJ[BII[BII)I */ -jint Java_org_rocksdb_RocksDB_get__JJ_3BII_3BII(JNIEnv* env, jobject, +jint Java_org_forstdb_RocksDB_get__JJ_3BII_3BII(JNIEnv* env, jobject, jlong jdb_handle, jlong jropt_handle, jbyteArray jkey, jint jkey_off, @@ -1720,11 +1760,11 @@ jint Java_org_rocksdb_RocksDB_get__JJ_3BII_3BII(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: get * Signature: (JJ[BII[BIIJ)I */ -jint Java_org_rocksdb_RocksDB_get__JJ_3BII_3BIIJ( +jint Java_org_forstdb_RocksDB_get__JJ_3BII_3BIIJ( JNIEnv* env, jobject, jlong jdb_handle, jlong jropt_handle, jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, jint jval_off, jint jval_len, jlong jcf_handle) { @@ -2110,11 +2150,11 @@ void multi_get_helper_direct(JNIEnv* env, jobject, ROCKSDB_NAMESPACE::DB* db, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: multiGet * Signature: (J[[B[I[I)[[B */ -jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B_3I_3I( +jobjectArray Java_org_forstdb_RocksDB_multiGet__J_3_3B_3I_3I( JNIEnv* env, jobject jdb, jlong jdb_handle, jobjectArray jkeys, jintArray jkey_offs, jintArray jkey_lens) { return multi_get_helper( @@ -2123,11 +2163,11 @@ jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B_3I_3I( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: multiGet * Signature: (J[[B[I[I[J)[[B */ -jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B_3I_3I_3J( +jobjectArray Java_org_forstdb_RocksDB_multiGet__J_3_3B_3I_3I_3J( JNIEnv* env, jobject jdb, jlong jdb_handle, jobjectArray jkeys, jintArray jkey_offs, jintArray jkey_lens, jlongArray jcolumn_family_handles) { @@ -2138,11 +2178,11 @@ jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B_3I_3I_3J( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: multiGet * Signature: (JJ[[B[I[I)[[B */ -jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B_3I_3I( +jobjectArray Java_org_forstdb_RocksDB_multiGet__JJ_3_3B_3I_3I( JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle, jobjectArray jkeys, jintArray jkey_offs, jintArray jkey_lens) { return multi_get_helper( @@ -2152,11 +2192,11 @@ jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B_3I_3I( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: multiGet * Signature: (JJ[[B[I[I[J)[[B */ -jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B_3I_3I_3J( +jobjectArray Java_org_forstdb_RocksDB_multiGet__JJ_3_3B_3I_3I_3J( JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle, jobjectArray jkeys, jintArray jkey_offs, jintArray jkey_lens, jlongArray jcolumn_family_handles) { @@ -2167,12 +2207,12 @@ jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B_3I_3I_3J( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: multiGet * Signature: - * (JJ[J[Ljava/nio/ByteBuffer;[I[I[Ljava/nio/ByteBuffer;[I[Lorg/rocksdb/Status;)V + * (JJ[J[Ljava/nio/ByteBuffer;[I[I[Ljava/nio/ByteBuffer;[I[Lorg/forstdb/Status;)V */ -void Java_org_rocksdb_RocksDB_multiGet__JJ_3J_3Ljava_nio_ByteBuffer_2_3I_3I_3Ljava_nio_ByteBuffer_2_3I_3Lorg_rocksdb_Status_2( +void Java_org_forstdb_RocksDB_multiGet__JJ_3J_3Ljava_nio_ByteBuffer_2_3I_3I_3Ljava_nio_ByteBuffer_2_3I_3Lorg_forstdb_Status_2( JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle, jlongArray jcolumn_family_handles, jobjectArray jkeys, jintArray jkey_offsets, jintArray jkey_lengths, jobjectArray jvalues, @@ -2271,12 +2311,114 @@ bool key_may_exist_direct_helper(JNIEnv* env, jlong jdb_handle, return exists; } +jboolean key_exists_helper(JNIEnv* env, jlong jdb_handle, jlong jcf_handle, + jlong jread_opts_handle, char* key, jint jkey_len) { + std::string value; + bool value_found = false; + + auto* db = reinterpret_cast(jdb_handle); + + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + ROCKSDB_NAMESPACE::ReadOptions read_opts = + jread_opts_handle == 0 + ? ROCKSDB_NAMESPACE::ReadOptions() + : *(reinterpret_cast( + jread_opts_handle)); + + ROCKSDB_NAMESPACE::Slice key_slice(key, jkey_len); + + const bool may_exist = + db->KeyMayExist(read_opts, cf_handle, key_slice, &value, &value_found); + + if (may_exist) { + ROCKSDB_NAMESPACE::Status s; + { + ROCKSDB_NAMESPACE::PinnableSlice pinnable_val; + s = db->Get(read_opts, cf_handle, key_slice, &pinnable_val); + } + if (s.IsNotFound()) { + return JNI_FALSE; + } else if (s.ok()) { + return JNI_TRUE; + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return JNI_FALSE; + } + } else { + return JNI_FALSE; + } +} + /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB + * Method: keyExist + * Signature: (JJJ[BII)Z + */ +jboolean Java_org_forstdb_RocksDB_keyExists(JNIEnv* env, jobject, + jlong jdb_handle, jlong jcf_handle, + jlong jread_opts_handle, + jbyteArray jkey, jint jkey_offset, + jint jkey_len) { + jbyte* key = new jbyte[jkey_len]; + env->GetByteArrayRegion(jkey, jkey_offset, jkey_len, key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key; + return JNI_FALSE; + } else { + jboolean key_exists = + key_exists_helper(env, jdb_handle, jcf_handle, jread_opts_handle, + reinterpret_cast(key), jkey_len); + delete[] key; + return key_exists; + } +} + +/* + private native boolean keyExistDirect(final long handle, final long + cfHandle, final long readOptHandle, final ByteBuffer key, final int keyOffset, + final int keyLength); + + + * Class: org_forstdb_RocksDB + * Method: keyExistDirect + * Signature: (JJJLjava/nio/ByteBuffer;II)Z + */ +jboolean Java_org_forstdb_RocksDB_keyExistsDirect( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jlong jread_opts_handle, jobject jkey, jint jkey_offset, jint jkey_len) { + char* key = reinterpret_cast(env->GetDirectBufferAddress(jkey)); + if (key == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid key argument (argument is not a valid direct ByteBuffer)"); + return JNI_FALSE; + } + if (env->GetDirectBufferCapacity(jkey) < (jkey_offset + jkey_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid key argument. Capacity is less than requested region (offset " + "+ length)."); + return JNI_FALSE; + } + + return key_exists_helper(env, jdb_handle, jcf_handle, jread_opts_handle, key, + jkey_len); +} + +/* + * Class: org_forstdb_RocksDB * Method: keyMayExist * Signature: (JJJ[BII)Z */ -jboolean Java_org_rocksdb_RocksDB_keyMayExist( +jboolean Java_org_forstdb_RocksDB_keyMayExist( JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jlong jread_opts_handle, jbyteArray jkey, jint jkey_offset, jint jkey_len) { bool has_exception = false; @@ -2296,11 +2438,11 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: keyMayExistDirect * Signature: (JJJLjava/nio/ByteBuffer;II)Z */ -jboolean Java_org_rocksdb_RocksDB_keyMayExistDirect( +jboolean Java_org_forstdb_RocksDB_keyMayExistDirect( JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jlong jread_opts_handle, jobject jkey, jint jkey_offset, jint jkey_len) { bool has_exception = false; @@ -2319,12 +2461,12 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExistDirect( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: keyMayExistDirectFoundValue * Signature: * (JJJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)[J */ -jintArray Java_org_rocksdb_RocksDB_keyMayExistDirectFoundValue( +jintArray Java_org_forstdb_RocksDB_keyMayExistDirectFoundValue( JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jlong jread_opts_handle, jobject jkey, jint jkey_offset, jint jkey_len, jobject jval, jint jval_offset, jint jval_len) { @@ -2391,11 +2533,11 @@ jintArray Java_org_rocksdb_RocksDB_keyMayExistDirectFoundValue( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: keyMayExistFoundValue * Signature: (JJJ[BII)[[B */ -jobjectArray Java_org_rocksdb_RocksDB_keyMayExistFoundValue( +jobjectArray Java_org_forstdb_RocksDB_keyMayExistFoundValue( JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jlong jread_opts_handle, jbyteArray jkey, jint jkey_offset, jint jkey_len) { bool has_exception = false; @@ -2479,65 +2621,27 @@ jobjectArray Java_org_rocksdb_RocksDB_keyMayExistFoundValue( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: iterator - * Signature: (J)J - */ -jlong Java_org_rocksdb_RocksDB_iterator__J(JNIEnv*, jobject, jlong db_handle) { - auto* db = reinterpret_cast(db_handle); - return rocksdb_iterator_helper(db, ROCKSDB_NAMESPACE::ReadOptions(), nullptr); -} - -/* - * Class: org_rocksdb_RocksDB - * Method: iterator - * Signature: (JJ)J - */ -jlong Java_org_rocksdb_RocksDB_iterator__JJ(JNIEnv*, jobject, jlong db_handle, - jlong jread_options_handle) { - auto* db = reinterpret_cast(db_handle); - auto& read_options = - *reinterpret_cast(jread_options_handle); - return rocksdb_iterator_helper(db, read_options, nullptr); -} - -/* - * Class: org_rocksdb_RocksDB - * Method: iteratorCF - * Signature: (JJ)J - */ -jlong Java_org_rocksdb_RocksDB_iteratorCF__JJ(JNIEnv*, jobject, jlong db_handle, - jlong jcf_handle) { - auto* db = reinterpret_cast(db_handle); - auto* cf_handle = - reinterpret_cast(jcf_handle); - return rocksdb_iterator_helper(db, ROCKSDB_NAMESPACE::ReadOptions(), - cf_handle); -} - -/* - * Class: org_rocksdb_RocksDB - * Method: iteratorCF * Signature: (JJJ)J */ -jlong Java_org_rocksdb_RocksDB_iteratorCF__JJJ(JNIEnv*, jobject, - jlong db_handle, - jlong jcf_handle, - jlong jread_options_handle) { +jlong Java_org_forstdb_RocksDB_iterator(JNIEnv*, jobject, jlong db_handle, + jlong jcf_handle, + jlong jread_options_handle) { auto* db = reinterpret_cast(db_handle); auto* cf_handle = reinterpret_cast(jcf_handle); auto& read_options = *reinterpret_cast(jread_options_handle); - return rocksdb_iterator_helper(db, read_options, cf_handle); + return GET_CPLUSPLUS_POINTER(db->NewIterator(read_options, cf_handle)); } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: iterators * Signature: (J[JJ)[J */ -jlongArray Java_org_rocksdb_RocksDB_iterators(JNIEnv* env, jobject, +jlongArray Java_org_forstdb_RocksDB_iterators(JNIEnv* env, jobject, jlong db_handle, jlongArray jcolumn_family_handles, jlong jread_options_handle) { @@ -2597,7 +2701,7 @@ jlongArray Java_org_rocksdb_RocksDB_iterators(JNIEnv* env, jobject, * Method: getSnapshot * Signature: (J)J */ -jlong Java_org_rocksdb_RocksDB_getSnapshot(JNIEnv*, jobject, jlong db_handle) { +jlong Java_org_forstdb_RocksDB_getSnapshot(JNIEnv*, jobject, jlong db_handle) { auto* db = reinterpret_cast(db_handle); const ROCKSDB_NAMESPACE::Snapshot* snapshot = db->GetSnapshot(); return GET_CPLUSPLUS_POINTER(snapshot); @@ -2607,7 +2711,7 @@ jlong Java_org_rocksdb_RocksDB_getSnapshot(JNIEnv*, jobject, jlong db_handle) { * Method: releaseSnapshot * Signature: (JJ)V */ -void Java_org_rocksdb_RocksDB_releaseSnapshot(JNIEnv*, jobject, jlong db_handle, +void Java_org_forstdb_RocksDB_releaseSnapshot(JNIEnv*, jobject, jlong db_handle, jlong snapshot_handle) { auto* db = reinterpret_cast(db_handle); auto* snapshot = @@ -2616,11 +2720,11 @@ void Java_org_rocksdb_RocksDB_releaseSnapshot(JNIEnv*, jobject, jlong db_handle, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getProperty * Signature: (JJLjava/lang/String;I)Ljava/lang/String; */ -jstring Java_org_rocksdb_RocksDB_getProperty(JNIEnv* env, jobject, +jstring Java_org_forstdb_RocksDB_getProperty(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jstring jproperty, jint jproperty_len) { @@ -2654,11 +2758,11 @@ jstring Java_org_rocksdb_RocksDB_getProperty(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getMapProperty * Signature: (JJLjava/lang/String;I)Ljava/util/Map; */ -jobject Java_org_rocksdb_RocksDB_getMapProperty(JNIEnv* env, jobject, +jobject Java_org_forstdb_RocksDB_getMapProperty(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jstring jproperty, @@ -2693,11 +2797,11 @@ jobject Java_org_rocksdb_RocksDB_getMapProperty(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getLongProperty * Signature: (JJLjava/lang/String;I)J */ -jlong Java_org_rocksdb_RocksDB_getLongProperty(JNIEnv* env, jobject, +jlong Java_org_forstdb_RocksDB_getLongProperty(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jstring jproperty, @@ -2732,21 +2836,21 @@ jlong Java_org_rocksdb_RocksDB_getLongProperty(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: resetStats * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_resetStats(JNIEnv*, jobject, jlong jdb_handle) { +void Java_org_forstdb_RocksDB_resetStats(JNIEnv*, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); db->ResetStats(); } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getAggregatedLongProperty * Signature: (JLjava/lang/String;I)J */ -jlong Java_org_rocksdb_RocksDB_getAggregatedLongProperty(JNIEnv* env, jobject, +jlong Java_org_forstdb_RocksDB_getAggregatedLongProperty(JNIEnv* env, jobject, jlong db_handle, jstring jproperty, jint jproperty_len) { @@ -2770,11 +2874,11 @@ jlong Java_org_rocksdb_RocksDB_getAggregatedLongProperty(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getApproximateSizes * Signature: (JJ[JB)[J */ -jlongArray Java_org_rocksdb_RocksDB_getApproximateSizes( +jlongArray Java_org_forstdb_RocksDB_getApproximateSizes( JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jlongArray jrange_slice_handles, jbyte jinclude_flags) { const jsize jlen = env->GetArrayLength(jrange_slice_handles); @@ -2849,11 +2953,11 @@ jlongArray Java_org_rocksdb_RocksDB_getApproximateSizes( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getApproximateMemTableStats * Signature: (JJJJ)[J */ -jlongArray Java_org_rocksdb_RocksDB_getApproximateMemTableStats( +jlongArray Java_org_forstdb_RocksDB_getApproximateMemTableStats( JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jlong jstartHandle, jlong jlimitHandle) { auto* start = reinterpret_cast(jstartHandle); @@ -2893,11 +2997,11 @@ jlongArray Java_org_rocksdb_RocksDB_getApproximateMemTableStats( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: compactRange * Signature: (J[BI[BIJJ)V */ -void Java_org_rocksdb_RocksDB_compactRange(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_compactRange(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jbegin, jint jbegin_len, jbyteArray jend, jint jend_len, @@ -2951,26 +3055,29 @@ void Java_org_rocksdb_RocksDB_compactRange(JNIEnv* env, jobject, } ROCKSDB_NAMESPACE::Status s; - if (jbegin_len > 0 || jend_len > 0) { - const ROCKSDB_NAMESPACE::Slice begin(str_begin); - const ROCKSDB_NAMESPACE::Slice end(str_end); - s = db->CompactRange(*compact_range_opts, cf_handle, &begin, &end); - } else { - s = db->CompactRange(*compact_range_opts, cf_handle, nullptr, nullptr); - } + std::unique_ptr begin; + std::unique_ptr end; + if (jbegin_len > 0) { + begin.reset(new ROCKSDB_NAMESPACE::Slice(str_begin)); + } + if (jend_len > 0) { + end.reset(new ROCKSDB_NAMESPACE::Slice(str_end)); + } + s = db->CompactRange(*compact_range_opts, cf_handle, begin.get(), end.get()); if (jcompact_range_opts_handle == 0) { delete compact_range_opts; } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: setOptions * Signature: (JJ[Ljava/lang/String;[Ljava/lang/String;)V */ -void Java_org_rocksdb_RocksDB_setOptions(JNIEnv* env, jobject, jlong jdb_handle, +void Java_org_forstdb_RocksDB_setOptions(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jobjectArray jkeys, jobjectArray jvalues) { const jsize len = env->GetArrayLength(jkeys); @@ -3029,11 +3136,11 @@ void Java_org_rocksdb_RocksDB_setOptions(JNIEnv* env, jobject, jlong jdb_handle, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: setDBOptions * Signature: (J[Ljava/lang/String;[Ljava/lang/String;)V */ -void Java_org_rocksdb_RocksDB_setDBOptions(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_setDBOptions(JNIEnv* env, jobject, jlong jdb_handle, jobjectArray jkeys, jobjectArray jvalues) { const jsize len = env->GetArrayLength(jkeys); @@ -3087,11 +3194,11 @@ void Java_org_rocksdb_RocksDB_setDBOptions(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getOptions * Signature: (JJ)Ljava/lang/String; */ -jstring Java_org_rocksdb_RocksDB_getOptions(JNIEnv* env, jobject, +jstring Java_org_forstdb_RocksDB_getOptions(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -3116,11 +3223,11 @@ jstring Java_org_rocksdb_RocksDB_getOptions(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getDBOptions * Signature: (J)Ljava/lang/String; */ -jstring Java_org_rocksdb_RocksDB_getDBOptions(JNIEnv* env, jobject, +jstring Java_org_forstdb_RocksDB_getDBOptions(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -3136,11 +3243,42 @@ jstring Java_org_rocksdb_RocksDB_getDBOptions(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB + * Method: setPerfLevel + * Signature: (JB)V + */ +void Java_org_forstdb_RocksDB_setPerfLevel(JNIEnv*, jobject, + jbyte jperf_level) { + ROCKSDB_NAMESPACE::SetPerfLevel( + ROCKSDB_NAMESPACE::PerfLevelTypeJni::toCppPerfLevelType(jperf_level)); +} + +/* + * Class: org_forstdb_RocksDB + * Method: getPerfLevel + * Signature: (J)B + */ +jbyte Java_org_forstdb_RocksDB_getPerfLevelNative(JNIEnv*, jobject) { + return ROCKSDB_NAMESPACE::PerfLevelTypeJni::toJavaPerfLevelType( + ROCKSDB_NAMESPACE::GetPerfLevel()); +} + +/* + * Class: org_forstdb_RocksDB + * Method: getPerfContextNative + * Signature: ()J + */ +jlong Java_org_forstdb_RocksDB_getPerfContextNative(JNIEnv*, jobject) { + ROCKSDB_NAMESPACE::PerfContext* perf_context = ROCKSDB_NAMESPACE::get_perf_context(); + return reinterpret_cast(perf_context); +} + +/* + * Class: org_forstdb_RocksDB * Method: compactFiles * Signature: (JJJ[Ljava/lang/String;IIJ)[Ljava/lang/String; */ -jobjectArray Java_org_rocksdb_RocksDB_compactFiles( +jobjectArray Java_org_forstdb_RocksDB_compactFiles( JNIEnv* env, jobject, jlong jdb_handle, jlong jcompaction_opts_handle, jlong jcf_handle, jobjectArray jinput_file_names, jint joutput_level, jint joutput_path_id, jlong jcompaction_job_info_handle) { @@ -3186,11 +3324,11 @@ jobjectArray Java_org_rocksdb_RocksDB_compactFiles( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: cancelAllBackgroundWork * Signature: (JZ)V */ -void Java_org_rocksdb_RocksDB_cancelAllBackgroundWork(JNIEnv*, jobject, +void Java_org_forstdb_RocksDB_cancelAllBackgroundWork(JNIEnv*, jobject, jlong jdb_handle, jboolean jwait) { auto* db = reinterpret_cast(jdb_handle); @@ -3198,11 +3336,11 @@ void Java_org_rocksdb_RocksDB_cancelAllBackgroundWork(JNIEnv*, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: pauseBackgroundWork * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_pauseBackgroundWork(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_pauseBackgroundWork(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); auto s = db->PauseBackgroundWork(); @@ -3212,11 +3350,11 @@ void Java_org_rocksdb_RocksDB_pauseBackgroundWork(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: continueBackgroundWork * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_continueBackgroundWork(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_continueBackgroundWork(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); auto s = db->ContinueBackgroundWork(); @@ -3226,11 +3364,11 @@ void Java_org_rocksdb_RocksDB_continueBackgroundWork(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: enableAutoCompaction * Signature: (J[J)V */ -void Java_org_rocksdb_RocksDB_enableAutoCompaction(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_enableAutoCompaction(JNIEnv* env, jobject, jlong jdb_handle, jlongArray jcf_handles) { auto* db = reinterpret_cast(jdb_handle); @@ -3247,11 +3385,11 @@ void Java_org_rocksdb_RocksDB_enableAutoCompaction(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: numberLevels * Signature: (JJ)I */ -jint Java_org_rocksdb_RocksDB_numberLevels(JNIEnv*, jobject, jlong jdb_handle, +jint Java_org_forstdb_RocksDB_numberLevels(JNIEnv*, jobject, jlong jdb_handle, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; @@ -3265,11 +3403,11 @@ jint Java_org_rocksdb_RocksDB_numberLevels(JNIEnv*, jobject, jlong jdb_handle, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: maxMemCompactionLevel * Signature: (JJ)I */ -jint Java_org_rocksdb_RocksDB_maxMemCompactionLevel(JNIEnv*, jobject, +jint Java_org_forstdb_RocksDB_maxMemCompactionLevel(JNIEnv*, jobject, jlong jdb_handle, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -3284,11 +3422,11 @@ jint Java_org_rocksdb_RocksDB_maxMemCompactionLevel(JNIEnv*, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: level0StopWriteTrigger * Signature: (JJ)I */ -jint Java_org_rocksdb_RocksDB_level0StopWriteTrigger(JNIEnv*, jobject, +jint Java_org_forstdb_RocksDB_level0StopWriteTrigger(JNIEnv*, jobject, jlong jdb_handle, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -3303,11 +3441,11 @@ jint Java_org_rocksdb_RocksDB_level0StopWriteTrigger(JNIEnv*, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getName * Signature: (J)Ljava/lang/String; */ -jstring Java_org_rocksdb_RocksDB_getName(JNIEnv* env, jobject, +jstring Java_org_forstdb_RocksDB_getName(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); std::string name = db->GetName(); @@ -3315,21 +3453,21 @@ jstring Java_org_rocksdb_RocksDB_getName(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getEnv * Signature: (J)J */ -jlong Java_org_rocksdb_RocksDB_getEnv(JNIEnv*, jobject, jlong jdb_handle) { +jlong Java_org_forstdb_RocksDB_getEnv(JNIEnv*, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); return GET_CPLUSPLUS_POINTER(db->GetEnv()); } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: flush * Signature: (JJ[J)V */ -void Java_org_rocksdb_RocksDB_flush(JNIEnv* env, jobject, jlong jdb_handle, +void Java_org_forstdb_RocksDB_flush(JNIEnv* env, jobject, jlong jdb_handle, jlong jflush_opts_handle, jlongArray jcf_handles) { auto* db = reinterpret_cast(jdb_handle); @@ -3355,11 +3493,11 @@ void Java_org_rocksdb_RocksDB_flush(JNIEnv* env, jobject, jlong jdb_handle, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: flushWal * Signature: (JZ)V */ -void Java_org_rocksdb_RocksDB_flushWal(JNIEnv* env, jobject, jlong jdb_handle, +void Java_org_forstdb_RocksDB_flushWal(JNIEnv* env, jobject, jlong jdb_handle, jboolean jsync) { auto* db = reinterpret_cast(jdb_handle); auto s = db->FlushWAL(jsync == JNI_TRUE); @@ -3369,11 +3507,11 @@ void Java_org_rocksdb_RocksDB_flushWal(JNIEnv* env, jobject, jlong jdb_handle, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: syncWal * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_syncWal(JNIEnv* env, jobject, jlong jdb_handle) { +void Java_org_forstdb_RocksDB_syncWal(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); auto s = db->SyncWAL(); if (!s.ok()) { @@ -3382,22 +3520,22 @@ void Java_org_rocksdb_RocksDB_syncWal(JNIEnv* env, jobject, jlong jdb_handle) { } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getLatestSequenceNumber * Signature: (J)V */ -jlong Java_org_rocksdb_RocksDB_getLatestSequenceNumber(JNIEnv*, jobject, +jlong Java_org_forstdb_RocksDB_getLatestSequenceNumber(JNIEnv*, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); return db->GetLatestSequenceNumber(); } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: disableFileDeletions * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_disableFileDeletions(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_disableFileDeletions(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); ROCKSDB_NAMESPACE::Status s = db->DisableFileDeletions(); @@ -3407,11 +3545,11 @@ void Java_org_rocksdb_RocksDB_disableFileDeletions(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: enableFileDeletions * Signature: (JZ)V */ -void Java_org_rocksdb_RocksDB_enableFileDeletions(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_enableFileDeletions(JNIEnv* env, jobject, jlong jdb_handle, jboolean jforce) { auto* db = reinterpret_cast(jdb_handle); @@ -3422,11 +3560,11 @@ void Java_org_rocksdb_RocksDB_enableFileDeletions(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getLiveFiles * Signature: (JZ)[Ljava/lang/String; */ -jobjectArray Java_org_rocksdb_RocksDB_getLiveFiles(JNIEnv* env, jobject, +jobjectArray Java_org_forstdb_RocksDB_getLiveFiles(JNIEnv* env, jobject, jlong jdb_handle, jboolean jflush_memtable) { auto* db = reinterpret_cast(jdb_handle); @@ -3447,11 +3585,11 @@ jobjectArray Java_org_rocksdb_RocksDB_getLiveFiles(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getSortedWalFiles - * Signature: (J)[Lorg/rocksdb/LogFile; + * Signature: (J)[Lorg/forstdb/LogFile; */ -jobjectArray Java_org_rocksdb_RocksDB_getSortedWalFiles(JNIEnv* env, jobject, +jobjectArray Java_org_forstdb_RocksDB_getSortedWalFiles(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); std::vector> sorted_wal_files; @@ -3495,11 +3633,11 @@ jobjectArray Java_org_rocksdb_RocksDB_getSortedWalFiles(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getUpdatesSince * Signature: (JJ)J */ -jlong Java_org_rocksdb_RocksDB_getUpdatesSince(JNIEnv* env, jobject, +jlong Java_org_forstdb_RocksDB_getUpdatesSince(JNIEnv* env, jobject, jlong jdb_handle, jlong jsequence_number) { auto* db = reinterpret_cast(jdb_handle); @@ -3516,11 +3654,11 @@ jlong Java_org_rocksdb_RocksDB_getUpdatesSince(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: deleteFile * Signature: (JLjava/lang/String;)V */ -void Java_org_rocksdb_RocksDB_deleteFile(JNIEnv* env, jobject, jlong jdb_handle, +void Java_org_forstdb_RocksDB_deleteFile(JNIEnv* env, jobject, jlong jdb_handle, jstring jname) { auto* db = reinterpret_cast(jdb_handle); jboolean has_exception = JNI_FALSE; @@ -3534,11 +3672,11 @@ void Java_org_rocksdb_RocksDB_deleteFile(JNIEnv* env, jobject, jlong jdb_handle, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getLiveFilesMetaData - * Signature: (J)[Lorg/rocksdb/LiveFileMetaData; + * Signature: (J)[Lorg/forstdb/LiveFileMetaData; */ -jobjectArray Java_org_rocksdb_RocksDB_getLiveFilesMetaData(JNIEnv* env, jobject, +jobjectArray Java_org_forstdb_RocksDB_getLiveFilesMetaData(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); std::vector live_files_meta_data; @@ -3581,11 +3719,11 @@ jobjectArray Java_org_rocksdb_RocksDB_getLiveFilesMetaData(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getColumnFamilyMetaData - * Signature: (JJ)Lorg/rocksdb/ColumnFamilyMetaData; + * Signature: (JJ)Lorg/forstdb/ColumnFamilyMetaData; */ -jobject Java_org_rocksdb_RocksDB_getColumnFamilyMetaData(JNIEnv* env, jobject, +jobject Java_org_forstdb_RocksDB_getColumnFamilyMetaData(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -3603,11 +3741,11 @@ jobject Java_org_rocksdb_RocksDB_getColumnFamilyMetaData(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: ingestExternalFile * Signature: (JJ[Ljava/lang/String;IJ)V */ -void Java_org_rocksdb_RocksDB_ingestExternalFile( +void Java_org_forstdb_RocksDB_ingestExternalFile( JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jobjectArray jfile_path_list, jint jfile_path_list_len, jlong jingest_external_file_options_handle) { @@ -3633,11 +3771,11 @@ void Java_org_rocksdb_RocksDB_ingestExternalFile( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: verifyChecksum * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_verifyChecksum(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_verifyChecksum(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); auto s = db->VerifyChecksum(); @@ -3647,11 +3785,11 @@ void Java_org_rocksdb_RocksDB_verifyChecksum(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getDefaultColumnFamily * Signature: (J)J */ -jlong Java_org_rocksdb_RocksDB_getDefaultColumnFamily(JNIEnv*, jobject, +jlong Java_org_forstdb_RocksDB_getDefaultColumnFamily(JNIEnv*, jobject, jlong jdb_handle) { auto* db_handle = reinterpret_cast(jdb_handle); auto* cf_handle = db_handle->DefaultColumnFamily(); @@ -3659,11 +3797,11 @@ jlong Java_org_rocksdb_RocksDB_getDefaultColumnFamily(JNIEnv*, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getPropertiesOfAllTables * Signature: (JJ)Ljava/util/Map; */ -jobject Java_org_rocksdb_RocksDB_getPropertiesOfAllTables(JNIEnv* env, jobject, +jobject Java_org_forstdb_RocksDB_getPropertiesOfAllTables(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -3731,11 +3869,11 @@ jobject Java_org_rocksdb_RocksDB_getPropertiesOfAllTables(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: getPropertiesOfTablesInRange * Signature: (JJ[J)Ljava/util/Map; */ -jobject Java_org_rocksdb_RocksDB_getPropertiesOfTablesInRange( +jobject Java_org_forstdb_RocksDB_getPropertiesOfTablesInRange( JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, jlongArray jrange_slice_handles) { auto* db = reinterpret_cast(jdb_handle); @@ -3784,11 +3922,11 @@ jobject Java_org_rocksdb_RocksDB_getPropertiesOfTablesInRange( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: suggestCompactRange * Signature: (JJ)[J */ -jlongArray Java_org_rocksdb_RocksDB_suggestCompactRange(JNIEnv* env, jobject, +jlongArray Java_org_forstdb_RocksDB_suggestCompactRange(JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -3834,11 +3972,11 @@ jlongArray Java_org_rocksdb_RocksDB_suggestCompactRange(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: promoteL0 * Signature: (JJI)V */ -void Java_org_rocksdb_RocksDB_promoteL0(JNIEnv*, jobject, jlong jdb_handle, +void Java_org_forstdb_RocksDB_promoteL0(JNIEnv*, jobject, jlong jdb_handle, jlong jcf_handle, jint jtarget_level) { auto* db = reinterpret_cast(jdb_handle); ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; @@ -3852,11 +3990,11 @@ void Java_org_rocksdb_RocksDB_promoteL0(JNIEnv*, jobject, jlong jdb_handle, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: startTrace * Signature: (JJJ)V */ -void Java_org_rocksdb_RocksDB_startTrace( +void Java_org_forstdb_RocksDB_startTrace( JNIEnv* env, jobject, jlong jdb_handle, jlong jmax_trace_file_size, jlong jtrace_writer_jnicallback_handle) { auto* db = reinterpret_cast(jdb_handle); @@ -3875,11 +4013,11 @@ void Java_org_rocksdb_RocksDB_startTrace( } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: endTrace * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_endTrace(JNIEnv* env, jobject, jlong jdb_handle) { +void Java_org_forstdb_RocksDB_endTrace(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); auto s = db->EndTrace(); if (!s.ok()) { @@ -3888,11 +4026,11 @@ void Java_org_rocksdb_RocksDB_endTrace(JNIEnv* env, jobject, jlong jdb_handle) { } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: tryCatchUpWithPrimary * Signature: (J)V */ -void Java_org_rocksdb_RocksDB_tryCatchUpWithPrimary(JNIEnv* env, jobject, +void Java_org_forstdb_RocksDB_tryCatchUpWithPrimary(JNIEnv* env, jobject, jlong jdb_handle) { auto* db = reinterpret_cast(jdb_handle); auto s = db->TryCatchUpWithPrimary(); @@ -3902,11 +4040,11 @@ void Java_org_rocksdb_RocksDB_tryCatchUpWithPrimary(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: destroyDB * Signature: (Ljava/lang/String;J)V */ -void Java_org_rocksdb_RocksDB_destroyDB(JNIEnv* env, jclass, jstring jdb_path, +void Java_org_forstdb_RocksDB_destroyDB(JNIEnv* env, jclass, jstring jdb_path, jlong joptions_handle) { const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); if (db_path == nullptr) { @@ -3957,11 +4095,11 @@ bool get_slice_helper(JNIEnv* env, jobjectArray ranges, jsize index, return true; } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: deleteFilesInRanges * Signature: (JJLjava/util/List;Z)V */ -void Java_org_rocksdb_RocksDB_deleteFilesInRanges(JNIEnv* env, jobject /*jdb*/, +void Java_org_forstdb_RocksDB_deleteFilesInRanges(JNIEnv* env, jobject /*jdb*/, jlong jdb_handle, jlong jcf_handle, jobjectArray ranges, @@ -4002,11 +4140,11 @@ void Java_org_rocksdb_RocksDB_deleteFilesInRanges(JNIEnv* env, jobject /*jdb*/, } /* - * Class: org_rocksdb_RocksDB + * Class: org_forstdb_RocksDB * Method: version * Signature: ()I */ -jint Java_org_rocksdb_RocksDB_version(JNIEnv*, jclass) { +jint Java_org_forstdb_RocksDB_version(JNIEnv*, jclass) { uint32_t encodedVersion = (ROCKSDB_MAJOR & 0xff) << 16; encodedVersion |= (ROCKSDB_MINOR & 0xff) << 8; encodedVersion |= (ROCKSDB_PATCH & 0xff); diff --git a/java/rocksjni/slice.cc b/java/forstjni/slice.cc similarity index 77% rename from java/rocksjni/slice.cc rename to java/forstjni/slice.cc index 63c6b1b9f..a72fa3f24 100644 --- a/java/rocksjni/slice.cc +++ b/java/forstjni/slice.cc @@ -14,20 +14,20 @@ #include -#include "include/org_rocksdb_AbstractSlice.h" -#include "include/org_rocksdb_DirectSlice.h" -#include "include/org_rocksdb_Slice.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_AbstractSlice.h" +#include "include/org_forstdb_DirectSlice.h" +#include "include/org_forstdb_Slice.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" -// /* - * Class: org_rocksdb_AbstractSlice + * Class: org_forstdb_AbstractSlice * Method: createNewSliceFromString * Signature: (Ljava/lang/String;)J */ -jlong Java_org_rocksdb_AbstractSlice_createNewSliceFromString(JNIEnv* env, +jlong Java_org_forstdb_AbstractSlice_createNewSliceFromString(JNIEnv* env, jclass /*jcls*/, jstring jstr) { const auto* str = env->GetStringUTFChars(jstr, nullptr); @@ -39,8 +39,8 @@ jlong Java_org_rocksdb_AbstractSlice_createNewSliceFromString(JNIEnv* env, const size_t len = strlen(str); // NOTE: buf will be deleted in the - // Java_org_rocksdb_Slice_disposeInternalBuf or - // or Java_org_rocksdb_DirectSlice_disposeInternalBuf methods + // Java_org_forstdb_Slice_disposeInternalBuf or + // or Java_org_forstdb_DirectSlice_disposeInternalBuf methods char* buf = new char[len + 1]; memcpy(buf, str, len); buf[len] = 0; @@ -51,33 +51,33 @@ jlong Java_org_rocksdb_AbstractSlice_createNewSliceFromString(JNIEnv* env, } /* - * Class: org_rocksdb_AbstractSlice + * Class: org_forstdb_AbstractSlice * Method: size0 * Signature: (J)I */ -jint Java_org_rocksdb_AbstractSlice_size0(JNIEnv* /*env*/, jobject /*jobj*/, +jint Java_org_forstdb_AbstractSlice_size0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { const auto* slice = reinterpret_cast(handle); return static_cast(slice->size()); } /* - * Class: org_rocksdb_AbstractSlice + * Class: org_forstdb_AbstractSlice * Method: empty0 * Signature: (J)Z */ -jboolean Java_org_rocksdb_AbstractSlice_empty0(JNIEnv* /*env*/, +jboolean Java_org_forstdb_AbstractSlice_empty0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { const auto* slice = reinterpret_cast(handle); return slice->empty(); } /* - * Class: org_rocksdb_AbstractSlice + * Class: org_forstdb_AbstractSlice * Method: toString0 * Signature: (JZ)Ljava/lang/String; */ -jstring Java_org_rocksdb_AbstractSlice_toString0(JNIEnv* env, jobject /*jobj*/, +jstring Java_org_forstdb_AbstractSlice_toString0(JNIEnv* env, jobject /*jobj*/, jlong handle, jboolean hex) { const auto* slice = reinterpret_cast(handle); const std::string s = slice->ToString(hex); @@ -85,11 +85,11 @@ jstring Java_org_rocksdb_AbstractSlice_toString0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_AbstractSlice + * Class: org_forstdb_AbstractSlice * Method: compare0 * Signature: (JJ)I; */ -jint Java_org_rocksdb_AbstractSlice_compare0(JNIEnv* /*env*/, jobject /*jobj*/, +jint Java_org_forstdb_AbstractSlice_compare0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jlong otherHandle) { const auto* slice = reinterpret_cast(handle); const auto* otherSlice = @@ -98,11 +98,11 @@ jint Java_org_rocksdb_AbstractSlice_compare0(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_AbstractSlice + * Class: org_forstdb_AbstractSlice * Method: startsWith0 * Signature: (JJ)Z; */ -jboolean Java_org_rocksdb_AbstractSlice_startsWith0(JNIEnv* /*env*/, +jboolean Java_org_forstdb_AbstractSlice_startsWith0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jlong otherHandle) { @@ -113,11 +113,11 @@ jboolean Java_org_rocksdb_AbstractSlice_startsWith0(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_AbstractSlice + * Class: org_forstdb_AbstractSlice * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_AbstractSlice_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_AbstractSlice_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { delete reinterpret_cast(handle); @@ -125,19 +125,19 @@ void Java_org_rocksdb_AbstractSlice_disposeInternal(JNIEnv* /*env*/, // -// /* - * Class: org_rocksdb_Slice + * Class: org_forstdb_Slice * Method: createNewSlice0 * Signature: ([BI)J */ -jlong Java_org_rocksdb_Slice_createNewSlice0(JNIEnv* env, jclass /*jcls*/, +jlong Java_org_forstdb_Slice_createNewSlice0(JNIEnv* env, jclass /*jcls*/, jbyteArray data, jint offset) { const jsize dataSize = env->GetArrayLength(data); const int len = dataSize - offset; - // NOTE: buf will be deleted in the Java_org_rocksdb_Slice_disposeInternalBuf + // NOTE: buf will be deleted in the Java_org_forstdb_Slice_disposeInternalBuf // method jbyte* buf = new jbyte[len]; env->GetByteArrayRegion(data, offset, len, buf); @@ -151,11 +151,11 @@ jlong Java_org_rocksdb_Slice_createNewSlice0(JNIEnv* env, jclass /*jcls*/, } /* - * Class: org_rocksdb_Slice + * Class: org_forstdb_Slice * Method: createNewSlice1 * Signature: ([B)J */ -jlong Java_org_rocksdb_Slice_createNewSlice1(JNIEnv* env, jclass /*jcls*/, +jlong Java_org_forstdb_Slice_createNewSlice1(JNIEnv* env, jclass /*jcls*/, jbyteArray data) { jbyte* ptrData = env->GetByteArrayElements(data, nullptr); if (ptrData == nullptr) { @@ -164,7 +164,7 @@ jlong Java_org_rocksdb_Slice_createNewSlice1(JNIEnv* env, jclass /*jcls*/, } const int len = env->GetArrayLength(data) + 1; - // NOTE: buf will be deleted in the Java_org_rocksdb_Slice_disposeInternalBuf + // NOTE: buf will be deleted in the Java_org_forstdb_Slice_disposeInternalBuf // method char* buf = new char[len]; memcpy(buf, ptrData, len - 1); @@ -178,11 +178,11 @@ jlong Java_org_rocksdb_Slice_createNewSlice1(JNIEnv* env, jclass /*jcls*/, } /* - * Class: org_rocksdb_Slice + * Class: org_forstdb_Slice * Method: data0 * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_Slice_data0(JNIEnv* env, jobject /*jobj*/, +jbyteArray Java_org_forstdb_Slice_data0(JNIEnv* env, jobject /*jobj*/, jlong handle) { const auto* slice = reinterpret_cast(handle); const jsize len = static_cast(slice->size()); @@ -205,11 +205,11 @@ jbyteArray Java_org_rocksdb_Slice_data0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Slice + * Class: org_forstdb_Slice * Method: clear0 * Signature: (JZJ)V */ -void Java_org_rocksdb_Slice_clear0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_Slice_clear0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jboolean shouldRelease, jlong internalBufferOffset) { auto* slice = reinterpret_cast(handle); @@ -221,33 +221,33 @@ void Java_org_rocksdb_Slice_clear0(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_Slice + * Class: org_forstdb_Slice * Method: removePrefix0 * Signature: (JI)V */ -void Java_org_rocksdb_Slice_removePrefix0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_Slice_removePrefix0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jint length) { auto* slice = reinterpret_cast(handle); slice->remove_prefix(length); } /* - * Class: org_rocksdb_DirectSlice + * Class: org_forstdb_DirectSlice * Method: setLength0 * Signature: (JI)V */ -void Java_org_rocksdb_DirectSlice_setLength0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_DirectSlice_setLength0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jint length) { auto* slice = reinterpret_cast(handle); slice->size_ = length; } /* - * Class: org_rocksdb_Slice + * Class: org_forstdb_Slice * Method: disposeInternalBuf * Signature: (JJ)V */ -void Java_org_rocksdb_Slice_disposeInternalBuf(JNIEnv* /*env*/, +void Java_org_forstdb_Slice_disposeInternalBuf(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jlong internalBufferOffset) { const auto* slice = reinterpret_cast(handle); @@ -257,14 +257,14 @@ void Java_org_rocksdb_Slice_disposeInternalBuf(JNIEnv* /*env*/, // -// /* - * Class: org_rocksdb_DirectSlice + * Class: org_forstdb_DirectSlice * Method: createNewDirectSlice0 * Signature: (Ljava/nio/ByteBuffer;I)J */ -jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice0(JNIEnv* env, +jlong Java_org_forstdb_DirectSlice_createNewDirectSlice0(JNIEnv* env, jclass /*jcls*/, jobject data, jint length) { @@ -285,11 +285,11 @@ jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice0(JNIEnv* env, } /* - * Class: org_rocksdb_DirectSlice + * Class: org_forstdb_DirectSlice * Method: createNewDirectSlice1 * Signature: (Ljava/nio/ByteBuffer;)J */ -jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice1(JNIEnv* env, +jlong Java_org_forstdb_DirectSlice_createNewDirectSlice1(JNIEnv* env, jclass /*jcls*/, jobject data) { void* data_addr = env->GetDirectBufferAddress(data); @@ -308,11 +308,11 @@ jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice1(JNIEnv* env, } /* - * Class: org_rocksdb_DirectSlice + * Class: org_forstdb_DirectSlice * Method: data0 * Signature: (J)Ljava/lang/Object; */ -jobject Java_org_rocksdb_DirectSlice_data0(JNIEnv* env, jobject /*jobj*/, +jobject Java_org_forstdb_DirectSlice_data0(JNIEnv* env, jobject /*jobj*/, jlong handle) { const auto* slice = reinterpret_cast(handle); return env->NewDirectByteBuffer(const_cast(slice->data()), @@ -320,22 +320,22 @@ jobject Java_org_rocksdb_DirectSlice_data0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_DirectSlice + * Class: org_forstdb_DirectSlice * Method: get0 * Signature: (JI)B */ -jbyte Java_org_rocksdb_DirectSlice_get0(JNIEnv* /*env*/, jobject /*jobj*/, +jbyte Java_org_forstdb_DirectSlice_get0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jint offset) { const auto* slice = reinterpret_cast(handle); return (*slice)[offset]; } /* - * Class: org_rocksdb_DirectSlice + * Class: org_forstdb_DirectSlice * Method: clear0 * Signature: (JZJ)V */ -void Java_org_rocksdb_DirectSlice_clear0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_DirectSlice_clear0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jboolean shouldRelease, jlong internalBufferOffset) { auto* slice = reinterpret_cast(handle); @@ -347,11 +347,11 @@ void Java_org_rocksdb_DirectSlice_clear0(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_DirectSlice + * Class: org_forstdb_DirectSlice * Method: removePrefix0 * Signature: (JI)V */ -void Java_org_rocksdb_DirectSlice_removePrefix0(JNIEnv* /*env*/, +void Java_org_forstdb_DirectSlice_removePrefix0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jint length) { auto* slice = reinterpret_cast(handle); @@ -359,11 +359,11 @@ void Java_org_rocksdb_DirectSlice_removePrefix0(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_DirectSlice + * Class: org_forstdb_DirectSlice * Method: disposeInternalBuf * Signature: (JJ)V */ -void Java_org_rocksdb_DirectSlice_disposeInternalBuf( +void Java_org_forstdb_DirectSlice_disposeInternalBuf( JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, jlong internalBufferOffset) { const auto* slice = reinterpret_cast(handle); diff --git a/java/rocksjni/snapshot.cc b/java/forstjni/snapshot.cc similarity index 81% rename from java/rocksjni/snapshot.cc rename to java/forstjni/snapshot.cc index 2a1265a58..f08f28a97 100644 --- a/java/rocksjni/snapshot.cc +++ b/java/forstjni/snapshot.cc @@ -9,16 +9,16 @@ #include #include -#include "include/org_rocksdb_Snapshot.h" +#include "include/org_forstdb_Snapshot.h" #include "rocksdb/db.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_Snapshot + * Class: org_forstdb_Snapshot * Method: getSequenceNumber * Signature: (J)J */ -jlong Java_org_rocksdb_Snapshot_getSequenceNumber(JNIEnv* /*env*/, +jlong Java_org_forstdb_Snapshot_getSequenceNumber(JNIEnv* /*env*/, jobject /*jobj*/, jlong jsnapshot_handle) { auto* snapshot = diff --git a/java/rocksjni/sst_file_manager.cc b/java/forstjni/sst_file_manager.cc similarity index 84% rename from java/rocksjni/sst_file_manager.cc rename to java/forstjni/sst_file_manager.cc index c51436819..4abe6f06f 100644 --- a/java/rocksjni/sst_file_manager.cc +++ b/java/forstjni/sst_file_manager.cc @@ -13,16 +13,16 @@ #include -#include "include/org_rocksdb_SstFileManager.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_SstFileManager.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: newSstFileManager * Signature: (JJJDJ)J */ -jlong Java_org_rocksdb_SstFileManager_newSstFileManager( +jlong Java_org_forstdb_SstFileManager_newSstFileManager( JNIEnv* jnienv, jclass /*jcls*/, jlong jenv_handle, jlong jlogger_handle, jlong jrate_bytes, jdouble jmax_trash_db_ratio, jlong jmax_delete_chunk_bytes) { @@ -56,11 +56,11 @@ jlong Java_org_rocksdb_SstFileManager_newSstFileManager( } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: setMaxAllowedSpaceUsage * Signature: (JJ)V */ -void Java_org_rocksdb_SstFileManager_setMaxAllowedSpaceUsage( +void Java_org_forstdb_SstFileManager_setMaxAllowedSpaceUsage( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jmax_allowed_space) { auto* sptr_sst_file_manager = @@ -70,11 +70,11 @@ void Java_org_rocksdb_SstFileManager_setMaxAllowedSpaceUsage( } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: setCompactionBufferSize * Signature: (JJ)V */ -void Java_org_rocksdb_SstFileManager_setCompactionBufferSize( +void Java_org_forstdb_SstFileManager_setCompactionBufferSize( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jcompaction_buffer_size) { auto* sptr_sst_file_manager = @@ -85,11 +85,11 @@ void Java_org_rocksdb_SstFileManager_setCompactionBufferSize( } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: isMaxAllowedSpaceReached * Signature: (J)Z */ -jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReached( +jboolean Java_org_forstdb_SstFileManager_isMaxAllowedSpaceReached( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_sst_file_manager = reinterpret_cast*>( @@ -98,12 +98,12 @@ jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReached( } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: isMaxAllowedSpaceReachedIncludingCompactions * Signature: (J)Z */ jboolean -Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReachedIncludingCompactions( +Java_org_forstdb_SstFileManager_isMaxAllowedSpaceReachedIncludingCompactions( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_sst_file_manager = reinterpret_cast*>( @@ -113,11 +113,11 @@ Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReachedIncludingCompactions( } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: getTotalSize * Signature: (J)J */ -jlong Java_org_rocksdb_SstFileManager_getTotalSize(JNIEnv* /*env*/, +jlong Java_org_forstdb_SstFileManager_getTotalSize(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_sst_file_manager = @@ -127,11 +127,11 @@ jlong Java_org_rocksdb_SstFileManager_getTotalSize(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: getTrackedFiles * Signature: (J)Ljava/util/Map; */ -jobject Java_org_rocksdb_SstFileManager_getTrackedFiles(JNIEnv* env, +jobject Java_org_forstdb_SstFileManager_getTrackedFiles(JNIEnv* env, jobject /*jobj*/, jlong jhandle) { auto* sptr_sst_file_manager = @@ -181,11 +181,11 @@ jobject Java_org_rocksdb_SstFileManager_getTrackedFiles(JNIEnv* env, } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: getDeleteRateBytesPerSecond * Signature: (J)J */ -jlong Java_org_rocksdb_SstFileManager_getDeleteRateBytesPerSecond( +jlong Java_org_forstdb_SstFileManager_getDeleteRateBytesPerSecond( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_sst_file_manager = reinterpret_cast*>( @@ -194,11 +194,11 @@ jlong Java_org_rocksdb_SstFileManager_getDeleteRateBytesPerSecond( } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: setDeleteRateBytesPerSecond * Signature: (JJ)V */ -void Java_org_rocksdb_SstFileManager_setDeleteRateBytesPerSecond( +void Java_org_forstdb_SstFileManager_setDeleteRateBytesPerSecond( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jdelete_rate) { auto* sptr_sst_file_manager = reinterpret_cast*>( @@ -207,11 +207,11 @@ void Java_org_rocksdb_SstFileManager_setDeleteRateBytesPerSecond( } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: getMaxTrashDBRatio * Signature: (J)D */ -jdouble Java_org_rocksdb_SstFileManager_getMaxTrashDBRatio(JNIEnv* /*env*/, +jdouble Java_org_forstdb_SstFileManager_getMaxTrashDBRatio(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_sst_file_manager = @@ -221,11 +221,11 @@ jdouble Java_org_rocksdb_SstFileManager_getMaxTrashDBRatio(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: setMaxTrashDBRatio * Signature: (JD)V */ -void Java_org_rocksdb_SstFileManager_setMaxTrashDBRatio(JNIEnv* /*env*/, +void Java_org_forstdb_SstFileManager_setMaxTrashDBRatio(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jdouble jratio) { @@ -236,11 +236,11 @@ void Java_org_rocksdb_SstFileManager_setMaxTrashDBRatio(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_SstFileManager + * Class: org_forstdb_SstFileManager * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_SstFileManager_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_SstFileManager_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* sptr_sst_file_manager = diff --git a/java/rocksjni/sst_file_reader_iterator.cc b/java/forstjni/sst_file_reader_iterator.cc similarity index 82% rename from java/rocksjni/sst_file_reader_iterator.cc rename to java/forstjni/sst_file_reader_iterator.cc index 68fa4c37c..71e33b781 100644 --- a/java/rocksjni/sst_file_reader_iterator.cc +++ b/java/forstjni/sst_file_reader_iterator.cc @@ -10,16 +10,16 @@ #include #include -#include "include/org_rocksdb_SstFileReaderIterator.h" +#include "include/org_forstdb_SstFileReaderIterator.h" #include "rocksdb/iterator.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); @@ -28,66 +28,66 @@ void Java_org_rocksdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: isValid0 * Signature: (J)Z */ -jboolean Java_org_rocksdb_SstFileReaderIterator_isValid0(JNIEnv* /*env*/, +jboolean Java_org_forstdb_SstFileReaderIterator_isValid0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { return reinterpret_cast(handle)->Valid(); } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: seekToFirst0 * Signature: (J)V */ -void Java_org_rocksdb_SstFileReaderIterator_seekToFirst0(JNIEnv* /*env*/, +void Java_org_forstdb_SstFileReaderIterator_seekToFirst0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->SeekToFirst(); } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: seekToLast0 * Signature: (J)V */ -void Java_org_rocksdb_SstFileReaderIterator_seekToLast0(JNIEnv* /*env*/, +void Java_org_forstdb_SstFileReaderIterator_seekToLast0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->SeekToLast(); } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: next0 * Signature: (J)V */ -void Java_org_rocksdb_SstFileReaderIterator_next0(JNIEnv* /*env*/, +void Java_org_forstdb_SstFileReaderIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->Next(); } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: prev0 * Signature: (J)V */ -void Java_org_rocksdb_SstFileReaderIterator_prev0(JNIEnv* /*env*/, +void Java_org_forstdb_SstFileReaderIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->Prev(); } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: seek0 * Signature: (J[BI)V */ -void Java_org_rocksdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_len) { @@ -107,11 +107,11 @@ void Java_org_rocksdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: seekForPrev0 * Signature: (J[BI)V */ -void Java_org_rocksdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env, +void Java_org_forstdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, @@ -132,11 +132,11 @@ void Java_org_rocksdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env, } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: status0 * Signature: (J)V */ -void Java_org_rocksdb_SstFileReaderIterator_status0(JNIEnv* env, +void Java_org_forstdb_SstFileReaderIterator_status0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); @@ -150,11 +150,11 @@ void Java_org_rocksdb_SstFileReaderIterator_status0(JNIEnv* env, } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: key0 * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_SstFileReaderIterator_key0(JNIEnv* env, +jbyteArray Java_org_forstdb_SstFileReaderIterator_key0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); @@ -172,11 +172,11 @@ jbyteArray Java_org_rocksdb_SstFileReaderIterator_key0(JNIEnv* env, } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: value0 * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_SstFileReaderIterator_value0(JNIEnv* env, +jbyteArray Java_org_forstdb_SstFileReaderIterator_value0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); @@ -195,11 +195,11 @@ jbyteArray Java_org_rocksdb_SstFileReaderIterator_value0(JNIEnv* env, } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: keyDirect0 * Signature: (JLjava/nio/ByteBuffer;II)I */ -jint Java_org_rocksdb_SstFileReaderIterator_keyDirect0( +jint Java_org_forstdb_SstFileReaderIterator_keyDirect0( JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -212,11 +212,11 @@ jint Java_org_rocksdb_SstFileReaderIterator_keyDirect0( * This method supports fetching into indirect byte buffers; * the Java wrapper extracts the byte[] and passes it here. * - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: keyByteArray0 * Signature: (J[BII)I */ -jint Java_org_rocksdb_SstFileReaderIterator_keyByteArray0( +jint Java_org_forstdb_SstFileReaderIterator_keyByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jkey, jint jkey_off, jint jkey_len) { auto* it = reinterpret_cast(handle); @@ -232,11 +232,11 @@ jint Java_org_rocksdb_SstFileReaderIterator_keyByteArray0( } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: valueDirect0 * Signature: (JLjava/nio/ByteBuffer;II)I */ -jint Java_org_rocksdb_SstFileReaderIterator_valueDirect0( +jint Java_org_forstdb_SstFileReaderIterator_valueDirect0( JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -249,11 +249,11 @@ jint Java_org_rocksdb_SstFileReaderIterator_valueDirect0( * This method supports fetching into indirect byte buffers; * the Java wrapper extracts the byte[] and passes it here. * - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: valueByteArray0 * Signature: (J[BII)I */ -jint Java_org_rocksdb_SstFileReaderIterator_valueByteArray0( +jint Java_org_forstdb_SstFileReaderIterator_valueByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jvalue_target, jint jvalue_off, jint jvalue_len) { auto* it = reinterpret_cast(handle); @@ -269,11 +269,11 @@ jint Java_org_rocksdb_SstFileReaderIterator_valueByteArray0( } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: seekDirect0 * Signature: (JLjava/nio/ByteBuffer;II)V */ -void Java_org_rocksdb_SstFileReaderIterator_seekDirect0( +void Java_org_forstdb_SstFileReaderIterator_seekDirect0( JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -285,11 +285,11 @@ void Java_org_rocksdb_SstFileReaderIterator_seekDirect0( } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: seekForPrevDirect0 * Signature: (JLjava/nio/ByteBuffer;II)V */ -void Java_org_rocksdb_SstFileReaderIterator_seekForPrevDirect0( +void Java_org_forstdb_SstFileReaderIterator_seekForPrevDirect0( JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -304,11 +304,11 @@ void Java_org_rocksdb_SstFileReaderIterator_seekForPrevDirect0( * This method supports fetching into indirect byte buffers; * the Java wrapper extracts the byte[] and passes it here. * - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: seekByteArray0 * Signature: (J[BII)V */ -void Java_org_rocksdb_SstFileReaderIterator_seekByteArray0( +void Java_org_forstdb_SstFileReaderIterator_seekByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_off, jint jtarget_len) { const std::unique_ptr target(new char[jtarget_len]); @@ -331,11 +331,11 @@ void Java_org_rocksdb_SstFileReaderIterator_seekByteArray0( * This method supports fetching into indirect byte buffers; * the Java wrapper extracts the byte[] and passes it here. * - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: seekForPrevByteArray0 * Signature: (J[BII)V */ -void Java_org_rocksdb_SstFileReaderIterator_seekForPrevByteArray0( +void Java_org_forstdb_SstFileReaderIterator_seekForPrevByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_off, jint jtarget_len) { const std::unique_ptr target(new char[jtarget_len]); @@ -355,11 +355,11 @@ void Java_org_rocksdb_SstFileReaderIterator_seekForPrevByteArray0( } /* - * Class: org_rocksdb_SstFileReaderIterator + * Class: org_forstdb_SstFileReaderIterator * Method: refresh0 * Signature: (J)V */ -void Java_org_rocksdb_SstFileReaderIterator_refresh0(JNIEnv* env, +void Java_org_forstdb_SstFileReaderIterator_refresh0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); diff --git a/java/rocksjni/sst_file_readerjni.cc b/java/forstjni/sst_file_readerjni.cc similarity index 82% rename from java/rocksjni/sst_file_readerjni.cc rename to java/forstjni/sst_file_readerjni.cc index 7ef711842..325ed6251 100644 --- a/java/rocksjni/sst_file_readerjni.cc +++ b/java/forstjni/sst_file_readerjni.cc @@ -11,20 +11,20 @@ #include -#include "include/org_rocksdb_SstFileReader.h" +#include "include/org_forstdb_SstFileReader.h" #include "rocksdb/comparator.h" #include "rocksdb/env.h" #include "rocksdb/options.h" #include "rocksdb/sst_file_reader.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_SstFileReader + * Class: org_forstdb_SstFileReader * Method: newSstFileReader * Signature: (J)J */ -jlong Java_org_rocksdb_SstFileReader_newSstFileReader(JNIEnv * /*env*/, +jlong Java_org_forstdb_SstFileReader_newSstFileReader(JNIEnv * /*env*/, jclass /*jcls*/, jlong joptions) { auto *options = @@ -35,11 +35,11 @@ jlong Java_org_rocksdb_SstFileReader_newSstFileReader(JNIEnv * /*env*/, } /* - * Class: org_rocksdb_SstFileReader + * Class: org_forstdb_SstFileReader * Method: open * Signature: (JLjava/lang/String;)V */ -void Java_org_rocksdb_SstFileReader_open(JNIEnv *env, jobject /*jobj*/, +void Java_org_forstdb_SstFileReader_open(JNIEnv *env, jobject /*jobj*/, jlong jhandle, jstring jfile_path) { const char *file_path = env->GetStringUTFChars(jfile_path, nullptr); if (file_path == nullptr) { @@ -57,11 +57,11 @@ void Java_org_rocksdb_SstFileReader_open(JNIEnv *env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileReader + * Class: org_forstdb_SstFileReader * Method: newIterator * Signature: (JJ)J */ -jlong Java_org_rocksdb_SstFileReader_newIterator(JNIEnv * /*env*/, +jlong Java_org_forstdb_SstFileReader_newIterator(JNIEnv * /*env*/, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle) { @@ -73,22 +73,22 @@ jlong Java_org_rocksdb_SstFileReader_newIterator(JNIEnv * /*env*/, } /* - * Class: org_rocksdb_SstFileReader + * Class: org_forstdb_SstFileReader * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_SstFileReader_disposeInternal(JNIEnv * /*env*/, +void Java_org_forstdb_SstFileReader_disposeInternal(JNIEnv * /*env*/, jobject /*jobj*/, jlong jhandle) { delete reinterpret_cast(jhandle); } /* - * Class: org_rocksdb_SstFileReader + * Class: org_forstdb_SstFileReader * Method: verifyChecksum * Signature: (J)V */ -void Java_org_rocksdb_SstFileReader_verifyChecksum(JNIEnv *env, +void Java_org_forstdb_SstFileReader_verifyChecksum(JNIEnv *env, jobject /*jobj*/, jlong jhandle) { auto *sst_file_reader = @@ -100,11 +100,11 @@ void Java_org_rocksdb_SstFileReader_verifyChecksum(JNIEnv *env, } /* - * Class: org_rocksdb_SstFileReader + * Class: org_forstdb_SstFileReader * Method: getTableProperties * Signature: (J)J */ -jobject Java_org_rocksdb_SstFileReader_getTableProperties(JNIEnv *env, +jobject Java_org_forstdb_SstFileReader_getTableProperties(JNIEnv *env, jobject /*jobj*/, jlong jhandle) { auto *sst_file_reader = diff --git a/java/rocksjni/sst_file_writerjni.cc b/java/forstjni/sst_file_writerjni.cc similarity index 86% rename from java/rocksjni/sst_file_writerjni.cc rename to java/forstjni/sst_file_writerjni.cc index 1898c3cfc..a21742228 100644 --- a/java/rocksjni/sst_file_writerjni.cc +++ b/java/forstjni/sst_file_writerjni.cc @@ -11,20 +11,20 @@ #include -#include "include/org_rocksdb_SstFileWriter.h" +#include "include/org_forstdb_SstFileWriter.h" #include "rocksdb/comparator.h" #include "rocksdb/env.h" #include "rocksdb/options.h" #include "rocksdb/sst_file_writer.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: newSstFileWriter * Signature: (JJJB)J */ -jlong Java_org_rocksdb_SstFileWriter_newSstFileWriter__JJJB( +jlong Java_org_forstdb_SstFileWriter_newSstFileWriter__JJJB( JNIEnv * /*env*/, jclass /*jcls*/, jlong jenvoptions, jlong joptions, jlong jcomparator_handle, jbyte jcomparator_type) { ROCKSDB_NAMESPACE::Comparator *comparator = nullptr; @@ -51,11 +51,11 @@ jlong Java_org_rocksdb_SstFileWriter_newSstFileWriter__JJJB( } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: newSstFileWriter * Signature: (JJ)J */ -jlong Java_org_rocksdb_SstFileWriter_newSstFileWriter__JJ(JNIEnv * /*env*/, +jlong Java_org_forstdb_SstFileWriter_newSstFileWriter__JJ(JNIEnv * /*env*/, jclass /*jcls*/, jlong jenvoptions, jlong joptions) { @@ -69,11 +69,11 @@ jlong Java_org_rocksdb_SstFileWriter_newSstFileWriter__JJ(JNIEnv * /*env*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: open * Signature: (JLjava/lang/String;)V */ -void Java_org_rocksdb_SstFileWriter_open(JNIEnv *env, jobject /*jobj*/, +void Java_org_forstdb_SstFileWriter_open(JNIEnv *env, jobject /*jobj*/, jlong jhandle, jstring jfile_path) { const char *file_path = env->GetStringUTFChars(jfile_path, nullptr); if (file_path == nullptr) { @@ -91,11 +91,11 @@ void Java_org_rocksdb_SstFileWriter_open(JNIEnv *env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: put * Signature: (JJJ)V */ -void Java_org_rocksdb_SstFileWriter_put__JJJ(JNIEnv *env, jobject /*jobj*/, +void Java_org_forstdb_SstFileWriter_put__JJJ(JNIEnv *env, jobject /*jobj*/, jlong jhandle, jlong jkey_handle, jlong jvalue_handle) { auto *key_slice = reinterpret_cast(jkey_handle); @@ -110,11 +110,11 @@ void Java_org_rocksdb_SstFileWriter_put__JJJ(JNIEnv *env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: put * Signature: (JJJ)V */ -void Java_org_rocksdb_SstFileWriter_put__J_3B_3B(JNIEnv *env, jobject /*jobj*/, +void Java_org_forstdb_SstFileWriter_put__J_3B_3B(JNIEnv *env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jbyteArray jval) { jbyte *key = env->GetByteArrayElements(jkey, nullptr); @@ -147,11 +147,11 @@ void Java_org_rocksdb_SstFileWriter_put__J_3B_3B(JNIEnv *env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: putDirect * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)V */ -void Java_org_rocksdb_SstFileWriter_putDirect(JNIEnv *env, jobject /*jdb*/, +void Java_org_forstdb_SstFileWriter_putDirect(JNIEnv *env, jobject /*jdb*/, jlong jdb_handle, jobject jkey, jint jkey_off, jint jkey_len, jobject jval, jint jval_off, @@ -171,11 +171,11 @@ void Java_org_rocksdb_SstFileWriter_putDirect(JNIEnv *env, jobject /*jdb*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: fileSize * Signature: (J)J */ -jlong Java_org_rocksdb_SstFileWriter_fileSize(JNIEnv * /*env*/, jobject /*jdb*/, +jlong Java_org_forstdb_SstFileWriter_fileSize(JNIEnv * /*env*/, jobject /*jdb*/, jlong jdb_handle) { auto *writer = reinterpret_cast(jdb_handle); @@ -183,11 +183,11 @@ jlong Java_org_rocksdb_SstFileWriter_fileSize(JNIEnv * /*env*/, jobject /*jdb*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: merge * Signature: (JJJ)V */ -void Java_org_rocksdb_SstFileWriter_merge__JJJ(JNIEnv *env, jobject /*jobj*/, +void Java_org_forstdb_SstFileWriter_merge__JJJ(JNIEnv *env, jobject /*jobj*/, jlong jhandle, jlong jkey_handle, jlong jvalue_handle) { auto *key_slice = reinterpret_cast(jkey_handle); @@ -202,11 +202,11 @@ void Java_org_rocksdb_SstFileWriter_merge__JJJ(JNIEnv *env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: merge * Signature: (J[B[B)V */ -void Java_org_rocksdb_SstFileWriter_merge__J_3B_3B(JNIEnv *env, +void Java_org_forstdb_SstFileWriter_merge__J_3B_3B(JNIEnv *env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, @@ -241,11 +241,11 @@ void Java_org_rocksdb_SstFileWriter_merge__J_3B_3B(JNIEnv *env, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: delete * Signature: (JJJ)V */ -void Java_org_rocksdb_SstFileWriter_delete__J_3B(JNIEnv *env, jobject /*jobj*/, +void Java_org_forstdb_SstFileWriter_delete__J_3B(JNIEnv *env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey) { jbyte *key = env->GetByteArrayElements(jkey, nullptr); @@ -268,11 +268,11 @@ void Java_org_rocksdb_SstFileWriter_delete__J_3B(JNIEnv *env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: delete * Signature: (JJJ)V */ -void Java_org_rocksdb_SstFileWriter_delete__JJ(JNIEnv *env, jobject /*jobj*/, +void Java_org_forstdb_SstFileWriter_delete__JJ(JNIEnv *env, jobject /*jobj*/, jlong jhandle, jlong jkey_handle) { auto *key_slice = reinterpret_cast(jkey_handle); @@ -285,11 +285,11 @@ void Java_org_rocksdb_SstFileWriter_delete__JJ(JNIEnv *env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: finish * Signature: (J)V */ -void Java_org_rocksdb_SstFileWriter_finish(JNIEnv *env, jobject /*jobj*/, +void Java_org_forstdb_SstFileWriter_finish(JNIEnv *env, jobject /*jobj*/, jlong jhandle) { ROCKSDB_NAMESPACE::Status s = reinterpret_cast(jhandle)->Finish(); @@ -299,11 +299,11 @@ void Java_org_rocksdb_SstFileWriter_finish(JNIEnv *env, jobject /*jobj*/, } /* - * Class: org_rocksdb_SstFileWriter + * Class: org_forstdb_SstFileWriter * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_SstFileWriter_disposeInternal(JNIEnv * /*env*/, +void Java_org_forstdb_SstFileWriter_disposeInternal(JNIEnv * /*env*/, jobject /*jobj*/, jlong jhandle) { delete reinterpret_cast(jhandle); diff --git a/java/rocksjni/sst_partitioner.cc b/java/forstjni/sst_partitioner.cc similarity index 74% rename from java/rocksjni/sst_partitioner.cc rename to java/forstjni/sst_partitioner.cc index 1cea3b0cb..f249cb13a 100644 --- a/java/rocksjni/sst_partitioner.cc +++ b/java/forstjni/sst_partitioner.cc @@ -13,17 +13,17 @@ #include -#include "include/org_rocksdb_SstPartitionerFixedPrefixFactory.h" +#include "include/org_forstdb_SstPartitionerFixedPrefixFactory.h" #include "rocksdb/sst_file_manager.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_SstPartitionerFixedPrefixFactory + * Class: org_forstdb_SstPartitionerFixedPrefixFactory * Method: newSstPartitionerFixedPrefixFactory0 * Signature: (J)J */ -jlong Java_org_rocksdb_SstPartitionerFixedPrefixFactory_newSstPartitionerFixedPrefixFactory0( +jlong Java_org_forstdb_SstPartitionerFixedPrefixFactory_newSstPartitionerFixedPrefixFactory0( JNIEnv*, jclass, jlong prefix_len) { auto* ptr = new std::shared_ptr( ROCKSDB_NAMESPACE::NewSstPartitionerFixedPrefixFactory(prefix_len)); @@ -31,11 +31,11 @@ jlong Java_org_rocksdb_SstPartitionerFixedPrefixFactory_newSstPartitionerFixedPr } /* - * Class: org_rocksdb_SstPartitionerFixedPrefixFactory + * Class: org_forstdb_SstPartitionerFixedPrefixFactory * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_SstPartitionerFixedPrefixFactory_disposeInternal( +void Java_org_forstdb_SstPartitionerFixedPrefixFactory_disposeInternal( JNIEnv*, jobject, jlong jhandle) { auto* ptr = reinterpret_cast< std::shared_ptr*>(jhandle); diff --git a/java/rocksjni/statistics.cc b/java/forstjni/statistics.cc similarity index 81% rename from java/rocksjni/statistics.cc rename to java/forstjni/statistics.cc index bd405afa1..eff45a3c3 100644 --- a/java/rocksjni/statistics.cc +++ b/java/forstjni/statistics.cc @@ -13,48 +13,48 @@ #include #include -#include "include/org_rocksdb_Statistics.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" -#include "rocksjni/statisticsjni.h" +#include "include/org_forstdb_Statistics.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" +#include "forstjni/statisticsjni.h" /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: newStatistics * Signature: ()J */ -jlong Java_org_rocksdb_Statistics_newStatistics__(JNIEnv* env, jclass jcls) { - return Java_org_rocksdb_Statistics_newStatistics___3BJ(env, jcls, nullptr, 0); +jlong Java_org_forstdb_Statistics_newStatistics__(JNIEnv* env, jclass jcls) { + return Java_org_forstdb_Statistics_newStatistics___3BJ(env, jcls, nullptr, 0); } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: newStatistics * Signature: (J)J */ -jlong Java_org_rocksdb_Statistics_newStatistics__J( +jlong Java_org_forstdb_Statistics_newStatistics__J( JNIEnv* env, jclass jcls, jlong jother_statistics_handle) { - return Java_org_rocksdb_Statistics_newStatistics___3BJ( + return Java_org_forstdb_Statistics_newStatistics___3BJ( env, jcls, nullptr, jother_statistics_handle); } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: newStatistics * Signature: ([B)J */ -jlong Java_org_rocksdb_Statistics_newStatistics___3B(JNIEnv* env, jclass jcls, +jlong Java_org_forstdb_Statistics_newStatistics___3B(JNIEnv* env, jclass jcls, jbyteArray jhistograms) { - return Java_org_rocksdb_Statistics_newStatistics___3BJ(env, jcls, jhistograms, + return Java_org_forstdb_Statistics_newStatistics___3BJ(env, jcls, jhistograms, 0); } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: newStatistics * Signature: ([BJ)J */ -jlong Java_org_rocksdb_Statistics_newStatistics___3BJ( +jlong Java_org_forstdb_Statistics_newStatistics___3BJ( JNIEnv* env, jclass, jbyteArray jhistograms, jlong jother_statistics_handle) { std::shared_ptr* pSptr_other_statistics = @@ -100,11 +100,11 @@ jlong Java_org_rocksdb_Statistics_newStatistics___3BJ( } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_Statistics_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_Statistics_disposeInternal(JNIEnv*, jobject, jlong jhandle) { if (jhandle > 0) { auto* pSptr_statistics = @@ -115,11 +115,11 @@ void Java_org_rocksdb_Statistics_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: statsLevel * Signature: (J)B */ -jbyte Java_org_rocksdb_Statistics_statsLevel(JNIEnv*, jobject, jlong jhandle) { +jbyte Java_org_forstdb_Statistics_statsLevel(JNIEnv*, jobject, jlong jhandle) { auto* pSptr_statistics = reinterpret_cast*>( jhandle); @@ -129,11 +129,11 @@ jbyte Java_org_rocksdb_Statistics_statsLevel(JNIEnv*, jobject, jlong jhandle) { } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: setStatsLevel * Signature: (JB)V */ -void Java_org_rocksdb_Statistics_setStatsLevel(JNIEnv*, jobject, jlong jhandle, +void Java_org_forstdb_Statistics_setStatsLevel(JNIEnv*, jobject, jlong jhandle, jbyte jstats_level) { auto* pSptr_statistics = reinterpret_cast*>( @@ -145,11 +145,11 @@ void Java_org_rocksdb_Statistics_setStatsLevel(JNIEnv*, jobject, jlong jhandle, } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: getTickerCount * Signature: (JB)J */ -jlong Java_org_rocksdb_Statistics_getTickerCount(JNIEnv*, jobject, +jlong Java_org_forstdb_Statistics_getTickerCount(JNIEnv*, jobject, jlong jhandle, jbyte jticker_type) { auto* pSptr_statistics = @@ -162,11 +162,11 @@ jlong Java_org_rocksdb_Statistics_getTickerCount(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: getAndResetTickerCount * Signature: (JB)J */ -jlong Java_org_rocksdb_Statistics_getAndResetTickerCount(JNIEnv*, jobject, +jlong Java_org_forstdb_Statistics_getAndResetTickerCount(JNIEnv*, jobject, jlong jhandle, jbyte jticker_type) { auto* pSptr_statistics = @@ -178,11 +178,11 @@ jlong Java_org_rocksdb_Statistics_getAndResetTickerCount(JNIEnv*, jobject, } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: getHistogramData - * Signature: (JB)Lorg/rocksdb/HistogramData; + * Signature: (JB)Lorg/forstdb/HistogramData; */ -jobject Java_org_rocksdb_Statistics_getHistogramData(JNIEnv* env, jobject, +jobject Java_org_forstdb_Statistics_getHistogramData(JNIEnv* env, jobject, jlong jhandle, jbyte jhistogram_type) { auto* pSptr_statistics = @@ -219,11 +219,11 @@ jobject Java_org_rocksdb_Statistics_getHistogramData(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: getHistogramString * Signature: (JB)Ljava/lang/String; */ -jstring Java_org_rocksdb_Statistics_getHistogramString(JNIEnv* env, jobject, +jstring Java_org_forstdb_Statistics_getHistogramString(JNIEnv* env, jobject, jlong jhandle, jbyte jhistogram_type) { auto* pSptr_statistics = @@ -237,11 +237,11 @@ jstring Java_org_rocksdb_Statistics_getHistogramString(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: reset * Signature: (J)V */ -void Java_org_rocksdb_Statistics_reset(JNIEnv* env, jobject, jlong jhandle) { +void Java_org_forstdb_Statistics_reset(JNIEnv* env, jobject, jlong jhandle) { auto* pSptr_statistics = reinterpret_cast*>( jhandle); @@ -253,11 +253,11 @@ void Java_org_rocksdb_Statistics_reset(JNIEnv* env, jobject, jlong jhandle) { } /* - * Class: org_rocksdb_Statistics + * Class: org_forstdb_Statistics * Method: toString * Signature: (J)Ljava/lang/String; */ -jstring Java_org_rocksdb_Statistics_toString(JNIEnv* env, jobject, +jstring Java_org_forstdb_Statistics_toString(JNIEnv* env, jobject, jlong jhandle) { auto* pSptr_statistics = reinterpret_cast*>( diff --git a/java/rocksjni/statisticsjni.cc b/java/forstjni/statisticsjni.cc similarity index 96% rename from java/rocksjni/statisticsjni.cc rename to java/forstjni/statisticsjni.cc index f46337893..4a053033f 100644 --- a/java/rocksjni/statisticsjni.cc +++ b/java/forstjni/statisticsjni.cc @@ -6,7 +6,7 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::Statistics -#include "rocksjni/statisticsjni.h" +#include "forstjni/statisticsjni.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/statisticsjni.h b/java/forstjni/statisticsjni.h similarity index 100% rename from java/rocksjni/statisticsjni.h rename to java/forstjni/statisticsjni.h diff --git a/java/rocksjni/table.cc b/java/forstjni/table.cc similarity index 94% rename from java/rocksjni/table.cc rename to java/forstjni/table.cc index 7f99900e4..b15c9777f 100644 --- a/java/rocksjni/table.cc +++ b/java/forstjni/table.cc @@ -10,19 +10,19 @@ #include -#include "include/org_rocksdb_BlockBasedTableConfig.h" -#include "include/org_rocksdb_PlainTableConfig.h" +#include "include/org_forstdb_BlockBasedTableConfig.h" +#include "include/org_forstdb_PlainTableConfig.h" #include "portal.h" #include "rocksdb/cache.h" #include "rocksdb/filter_policy.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_PlainTableConfig + * Class: org_forstdb_PlainTableConfig * Method: newTableFactoryHandle * Signature: (IIDIIBZZ)J */ -jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle( +jlong Java_org_forstdb_PlainTableConfig_newTableFactoryHandle( JNIEnv * /*env*/, jobject /*jobj*/, jint jkey_size, jint jbloom_bits_per_key, jdouble jhash_table_ratio, jint jindex_sparseness, jint jhuge_page_tlb_size, jbyte jencoding_type, jboolean jfull_scan_mode, @@ -43,11 +43,11 @@ jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle( } /* - * Class: org_rocksdb_BlockBasedTableConfig + * Class: org_forstdb_BlockBasedTableConfig * Method: newTableFactoryHandle * Signature: (ZZZZBBDBZJJJJIIIJZZZJZZIIZZBJIJI)J */ -jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle( +jlong Java_org_forstdb_BlockBasedTableConfig_newTableFactoryHandle( JNIEnv *, jobject, jboolean jcache_index_and_filter_blocks, jboolean jcache_index_and_filter_blocks_with_high_priority, jboolean jpin_l0_filter_and_index_blocks_in_cache, diff --git a/java/rocksjni/table_filter.cc b/java/forstjni/table_filter.cc similarity index 68% rename from java/rocksjni/table_filter.cc rename to java/forstjni/table_filter.cc index 1400fa1d9..208aca21a 100644 --- a/java/rocksjni/table_filter.cc +++ b/java/forstjni/table_filter.cc @@ -4,22 +4,22 @@ // (found in the LICENSE.Apache file in the root directory). // // This file implements the "bridge" between Java and C++ for -// org.rocksdb.AbstractTableFilter. +// org.forstdb.AbstractTableFilter. #include #include -#include "include/org_rocksdb_AbstractTableFilter.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/table_filter_jnicallback.h" +#include "include/org_forstdb_AbstractTableFilter.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/table_filter_jnicallback.h" /* - * Class: org_rocksdb_AbstractTableFilter + * Class: org_forstdb_AbstractTableFilter * Method: createNewTableFilter * Signature: ()J */ -jlong Java_org_rocksdb_AbstractTableFilter_createNewTableFilter( +jlong Java_org_forstdb_AbstractTableFilter_createNewTableFilter( JNIEnv* env, jobject jtable_filter) { auto* table_filter_jnicallback = new ROCKSDB_NAMESPACE::TableFilterJniCallback(env, jtable_filter); diff --git a/java/rocksjni/table_filter_jnicallback.cc b/java/forstjni/table_filter_jnicallback.cc similarity index 96% rename from java/rocksjni/table_filter_jnicallback.cc rename to java/forstjni/table_filter_jnicallback.cc index 5350c5cee..061919f7b 100644 --- a/java/rocksjni/table_filter_jnicallback.cc +++ b/java/forstjni/table_filter_jnicallback.cc @@ -6,9 +6,9 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::TableFilter. -#include "rocksjni/table_filter_jnicallback.h" +#include "forstjni/table_filter_jnicallback.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { TableFilterJniCallback::TableFilterJniCallback(JNIEnv* env, diff --git a/java/rocksjni/table_filter_jnicallback.h b/java/forstjni/table_filter_jnicallback.h similarity index 96% rename from java/rocksjni/table_filter_jnicallback.h rename to java/forstjni/table_filter_jnicallback.h index 0ef404ca2..7fbec5994 100644 --- a/java/rocksjni/table_filter_jnicallback.h +++ b/java/forstjni/table_filter_jnicallback.h @@ -15,7 +15,7 @@ #include #include "rocksdb/table_properties.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/testable_event_listener.cc b/java/forstjni/testable_event_listener.cc similarity index 98% rename from java/rocksjni/testable_event_listener.cc rename to java/forstjni/testable_event_listener.cc index 71188bc3c..a9380d440 100644 --- a/java/rocksjni/testable_event_listener.cc +++ b/java/forstjni/testable_event_listener.cc @@ -7,7 +7,7 @@ #include #include -#include "include/org_rocksdb_test_TestableEventListener.h" +#include "include/org_forstdb_test_TestableEventListener.h" #include "rocksdb/listener.h" #include "rocksdb/status.h" #include "rocksdb/table_properties.h" @@ -73,11 +73,11 @@ static TableProperties newTablePropertiesForTest() { } /* - * Class: org_rocksdb_test_TestableEventListener + * Class: org_forstdb_test_TestableEventListener * Method: invokeAllCallbacks * Signature: (J)V */ -void Java_org_rocksdb_test_TestableEventListener_invokeAllCallbacks( +void Java_org_forstdb_test_TestableEventListener_invokeAllCallbacks( JNIEnv *, jclass, jlong jhandle) { const auto &el = *reinterpret_cast *>( diff --git a/java/rocksjni/thread_status.cc b/java/forstjni/thread_status.cc similarity index 83% rename from java/rocksjni/thread_status.cc rename to java/forstjni/thread_status.cc index c600f6cd5..312e8c590 100644 --- a/java/rocksjni/thread_status.cc +++ b/java/forstjni/thread_status.cc @@ -10,15 +10,15 @@ #include -#include "include/org_rocksdb_ThreadStatus.h" +#include "include/org_forstdb_ThreadStatus.h" #include "portal.h" /* - * Class: org_rocksdb_ThreadStatus + * Class: org_forstdb_ThreadStatus * Method: getThreadTypeName * Signature: (B)Ljava/lang/String; */ -jstring Java_org_rocksdb_ThreadStatus_getThreadTypeName( +jstring Java_org_forstdb_ThreadStatus_getThreadTypeName( JNIEnv* env, jclass, jbyte jthread_type_value) { auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetThreadTypeName( ROCKSDB_NAMESPACE::ThreadTypeJni::toCppThreadType(jthread_type_value)); @@ -26,11 +26,11 @@ jstring Java_org_rocksdb_ThreadStatus_getThreadTypeName( } /* - * Class: org_rocksdb_ThreadStatus + * Class: org_forstdb_ThreadStatus * Method: getOperationName * Signature: (B)Ljava/lang/String; */ -jstring Java_org_rocksdb_ThreadStatus_getOperationName( +jstring Java_org_forstdb_ThreadStatus_getOperationName( JNIEnv* env, jclass, jbyte joperation_type_value) { auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetOperationName( ROCKSDB_NAMESPACE::OperationTypeJni::toCppOperationType( @@ -39,11 +39,11 @@ jstring Java_org_rocksdb_ThreadStatus_getOperationName( } /* - * Class: org_rocksdb_ThreadStatus + * Class: org_forstdb_ThreadStatus * Method: microsToStringNative * Signature: (J)Ljava/lang/String; */ -jstring Java_org_rocksdb_ThreadStatus_microsToStringNative(JNIEnv* env, jclass, +jstring Java_org_forstdb_ThreadStatus_microsToStringNative(JNIEnv* env, jclass, jlong jmicros) { auto str = ROCKSDB_NAMESPACE::ThreadStatus::MicrosToString( static_cast(jmicros)); @@ -51,11 +51,11 @@ jstring Java_org_rocksdb_ThreadStatus_microsToStringNative(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_ThreadStatus + * Class: org_forstdb_ThreadStatus * Method: getOperationStageName * Signature: (B)Ljava/lang/String; */ -jstring Java_org_rocksdb_ThreadStatus_getOperationStageName( +jstring Java_org_forstdb_ThreadStatus_getOperationStageName( JNIEnv* env, jclass, jbyte joperation_stage_value) { auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetOperationStageName( ROCKSDB_NAMESPACE::OperationStageJni::toCppOperationStage( @@ -64,11 +64,11 @@ jstring Java_org_rocksdb_ThreadStatus_getOperationStageName( } /* - * Class: org_rocksdb_ThreadStatus + * Class: org_forstdb_ThreadStatus * Method: getOperationPropertyName * Signature: (BI)Ljava/lang/String; */ -jstring Java_org_rocksdb_ThreadStatus_getOperationPropertyName( +jstring Java_org_forstdb_ThreadStatus_getOperationPropertyName( JNIEnv* env, jclass, jbyte joperation_type_value, jint jindex) { auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetOperationPropertyName( ROCKSDB_NAMESPACE::OperationTypeJni::toCppOperationType( @@ -78,11 +78,11 @@ jstring Java_org_rocksdb_ThreadStatus_getOperationPropertyName( } /* - * Class: org_rocksdb_ThreadStatus + * Class: org_forstdb_ThreadStatus * Method: interpretOperationProperties * Signature: (B[J)Ljava/util/Map; */ -jobject Java_org_rocksdb_ThreadStatus_interpretOperationProperties( +jobject Java_org_forstdb_ThreadStatus_interpretOperationProperties( JNIEnv* env, jclass, jbyte joperation_type_value, jlongArray joperation_properties) { // convert joperation_properties @@ -113,11 +113,11 @@ jobject Java_org_rocksdb_ThreadStatus_interpretOperationProperties( } /* - * Class: org_rocksdb_ThreadStatus + * Class: org_forstdb_ThreadStatus * Method: getStateName * Signature: (B)Ljava/lang/String; */ -jstring Java_org_rocksdb_ThreadStatus_getStateName(JNIEnv* env, jclass, +jstring Java_org_forstdb_ThreadStatus_getStateName(JNIEnv* env, jclass, jbyte jstate_type_value) { auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetStateName( ROCKSDB_NAMESPACE::StateTypeJni::toCppStateType(jstate_type_value)); diff --git a/java/rocksjni/trace_writer.cc b/java/forstjni/trace_writer.cc similarity index 72% rename from java/rocksjni/trace_writer.cc rename to java/forstjni/trace_writer.cc index d58276399..8152a0b17 100644 --- a/java/rocksjni/trace_writer.cc +++ b/java/forstjni/trace_writer.cc @@ -8,16 +8,16 @@ #include -#include "include/org_rocksdb_AbstractTraceWriter.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/trace_writer_jnicallback.h" +#include "include/org_forstdb_AbstractTraceWriter.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/trace_writer_jnicallback.h" /* - * Class: org_rocksdb_AbstractTraceWriter + * Class: org_forstdb_AbstractTraceWriter * Method: createNewTraceWriter * Signature: ()J */ -jlong Java_org_rocksdb_AbstractTraceWriter_createNewTraceWriter(JNIEnv* env, +jlong Java_org_forstdb_AbstractTraceWriter_createNewTraceWriter(JNIEnv* env, jobject jobj) { auto* trace_writer = new ROCKSDB_NAMESPACE::TraceWriterJniCallback(env, jobj); return GET_CPLUSPLUS_POINTER(trace_writer); diff --git a/java/rocksjni/trace_writer_jnicallback.cc b/java/forstjni/trace_writer_jnicallback.cc similarity index 97% rename from java/rocksjni/trace_writer_jnicallback.cc rename to java/forstjni/trace_writer_jnicallback.cc index d1ed32038..88934f269 100644 --- a/java/rocksjni/trace_writer_jnicallback.cc +++ b/java/forstjni/trace_writer_jnicallback.cc @@ -6,9 +6,9 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::TraceWriter. -#include "rocksjni/trace_writer_jnicallback.h" +#include "forstjni/trace_writer_jnicallback.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { TraceWriterJniCallback::TraceWriterJniCallback(JNIEnv* env, diff --git a/java/rocksjni/trace_writer_jnicallback.h b/java/forstjni/trace_writer_jnicallback.h similarity index 96% rename from java/rocksjni/trace_writer_jnicallback.h rename to java/forstjni/trace_writer_jnicallback.h index c82a3a72c..bb9ee895d 100644 --- a/java/rocksjni/trace_writer_jnicallback.h +++ b/java/forstjni/trace_writer_jnicallback.h @@ -14,7 +14,7 @@ #include #include "rocksdb/trace_reader_writer.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/transaction.cc b/java/forstjni/transaction.cc similarity index 68% rename from java/rocksjni/transaction.cc rename to java/forstjni/transaction.cc index 1a0a64fc7..7b64a7458 100644 --- a/java/rocksjni/transaction.cc +++ b/java/forstjni/transaction.cc @@ -12,9 +12,10 @@ #include -#include "include/org_rocksdb_Transaction.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_Transaction.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/kv_helper.h" +#include "forstjni/portal.h" #if defined(_MSC_VER) #pragma warning(push) @@ -23,33 +24,33 @@ #endif /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: setSnapshot * Signature: (J)V */ -void Java_org_rocksdb_Transaction_setSnapshot(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_Transaction_setSnapshot(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); txn->SetSnapshot(); } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: setSnapshotOnNextOperation * Signature: (J)V */ -void Java_org_rocksdb_Transaction_setSnapshotOnNextOperation__J( +void Java_org_forstdb_Transaction_setSnapshotOnNextOperation__J( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); txn->SetSnapshotOnNextOperation(nullptr); } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: setSnapshotOnNextOperation * Signature: (JJ)V */ -void Java_org_rocksdb_Transaction_setSnapshotOnNextOperation__JJ( +void Java_org_forstdb_Transaction_setSnapshotOnNextOperation__JJ( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jtxn_notifier_handle) { auto* txn = reinterpret_cast(jhandle); @@ -60,11 +61,11 @@ void Java_org_rocksdb_Transaction_setSnapshotOnNextOperation__JJ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getSnapshot * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getSnapshot(JNIEnv* /*env*/, +jlong Java_org_forstdb_Transaction_getSnapshot(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -73,11 +74,11 @@ jlong Java_org_rocksdb_Transaction_getSnapshot(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: clearSnapshot * Signature: (J)V */ -void Java_org_rocksdb_Transaction_clearSnapshot(JNIEnv* /*env*/, +void Java_org_forstdb_Transaction_clearSnapshot(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -85,11 +86,11 @@ void Java_org_rocksdb_Transaction_clearSnapshot(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: prepare * Signature: (J)V */ -void Java_org_rocksdb_Transaction_prepare(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_Transaction_prepare(JNIEnv* env, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); ROCKSDB_NAMESPACE::Status s = txn->Prepare(); @@ -99,11 +100,11 @@ void Java_org_rocksdb_Transaction_prepare(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: commit * Signature: (J)V */ -void Java_org_rocksdb_Transaction_commit(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_Transaction_commit(JNIEnv* env, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); ROCKSDB_NAMESPACE::Status s = txn->Commit(); @@ -113,11 +114,11 @@ void Java_org_rocksdb_Transaction_commit(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: rollback * Signature: (J)V */ -void Java_org_rocksdb_Transaction_rollback(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_Transaction_rollback(JNIEnv* env, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); ROCKSDB_NAMESPACE::Status s = txn->Rollback(); @@ -127,11 +128,11 @@ void Java_org_rocksdb_Transaction_rollback(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: setSavePoint * Signature: (J)V */ -void Java_org_rocksdb_Transaction_setSavePoint(JNIEnv* /*env*/, +void Java_org_forstdb_Transaction_setSavePoint(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -139,11 +140,11 @@ void Java_org_rocksdb_Transaction_setSavePoint(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: rollbackToSavePoint * Signature: (J)V */ -void Java_org_rocksdb_Transaction_rollbackToSavePoint(JNIEnv* env, +void Java_org_forstdb_Transaction_rollbackToSavePoint(JNIEnv* env, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -158,93 +159,116 @@ typedef std::function FnGet; -// TODO(AR) consider refactoring to share this between here and rocksjni.cc -jbyteArray txn_get_helper(JNIEnv* env, const FnGet& fn_get, - const jlong& jread_options_handle, - const jbyteArray& jkey, const jint& jkey_part_len) { - jbyte* key = env->GetByteArrayElements(jkey, nullptr); - if (key == nullptr) { - // exception thrown: OutOfMemoryError +/* + * Class: org_forstdb_Transaction + * Method: get + * Signature: (JJ[BIIJ)[B + */ +jbyteArray Java_org_forstdb_Transaction_get__JJ_3BIIJ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jbyteArray jkey, jint jkey_off, jint jkey_part_len, + jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* read_options = + reinterpret_cast(jread_options_handle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArrayPinnableSlice value(env); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Get(*read_options, column_family_handle, key.slice(), + &value.pinnable_slice())); + return value.NewByteArray(); + } catch (ROCKSDB_NAMESPACE::KVException&) { return nullptr; } - ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), - jkey_part_len); +} +/* + * Class: org_forstdb_Transaction + * Method: get + * Signature: (JJ[BII)[B + */ +jbyteArray Java_org_forstdb_Transaction_get__JJ_3BII( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jbyteArray jkey, jint jkey_off, jint jkey_part_len) { + auto* txn = reinterpret_cast(jhandle); auto* read_options = reinterpret_cast(jread_options_handle); - std::string value; - ROCKSDB_NAMESPACE::Status s = fn_get(*read_options, key_slice, &value); - - // trigger java unref on key. - // by passing JNI_ABORT, it will simply release the reference without - // copying the result back to the java byte array. - env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); - - if (s.IsNotFound()) { + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArrayPinnableSlice value(env); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Get(*read_options, key.slice(), &value.pinnable_slice())); + return value.NewByteArray(); + } catch (ROCKSDB_NAMESPACE::KVException&) { return nullptr; } - - if (s.ok()) { - jbyteArray jret_value = env->NewByteArray(static_cast(value.size())); - if (jret_value == nullptr) { - // exception thrown: OutOfMemoryError - return nullptr; - } - env->SetByteArrayRegion( - jret_value, 0, static_cast(value.size()), - const_cast(reinterpret_cast(value.c_str()))); - if (env->ExceptionCheck()) { - // exception thrown: ArrayIndexOutOfBoundsException - return nullptr; - } - return jret_value; - } - - ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); - return nullptr; } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: get - * Signature: (JJ[BIJ)[B + * Signature: (JJ[BII[BIIJ)I */ -jbyteArray Java_org_rocksdb_Transaction_get__JJ_3BIJ( +jint Java_org_forstdb_Transaction_get__JJ_3BII_3BIIJ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, - jbyteArray jkey, jint jkey_part_len, jlong jcolumn_family_handle) { + jbyteArray jkey, jint jkey_off, jint jkey_part_len, jbyteArray jval, + jint jval_off, jint jval_part_len, jlong jcolumn_family_handle) { auto* txn = reinterpret_cast(jhandle); + auto* read_options = + reinterpret_cast(jread_options_handle); auto* column_family_handle = reinterpret_cast( jcolumn_family_handle); - FnGet fn_get = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::Get, txn, std::placeholders::_1, - column_family_handle, std::placeholders::_2, std::placeholders::_3); - return txn_get_helper(env, fn_get, jread_options_handle, jkey, jkey_part_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArrayPinnableSlice value(env, jval, jval_off, + jval_part_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Get(*read_options, column_family_handle, key.slice(), + &value.pinnable_slice())); + return value.Fetch(); + } catch (const ROCKSDB_NAMESPACE::KVException& e) { + return e.Code(); + } } /* - * Class: org_rocksdb_Transaction - * Method: get - * Signature: (JJ[BI)[B + * Class: org_forstdb_Transaction + * Method: getDirect + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)I */ -jbyteArray Java_org_rocksdb_Transaction_get__JJ_3BI( - JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, - jbyteArray jkey, jint jkey_part_len) { +jint Java_org_forstdb_Transaction_getDirect(JNIEnv* env, jobject, jlong jhandle, + jlong jread_options_handle, + jobject jkey_bb, jint jkey_off, + jint jkey_part_len, jobject jval_bb, + jint jval_off, jint jval_part_len, + jlong jcolumn_family_handle) { auto* txn = reinterpret_cast(jhandle); - FnGet fn_get = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::Get, txn, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3); - return txn_get_helper(env, fn_get, jread_options_handle, jkey, jkey_part_len); + auto* read_options = + reinterpret_cast(jread_options_handle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + + try { + ROCKSDB_NAMESPACE::JDirectBufferSlice key(env, jkey_bb, jkey_off, + jkey_part_len); + ROCKSDB_NAMESPACE::JDirectBufferPinnableSlice value(env, jval_bb, jval_off, + jval_part_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Get(*read_options, column_family_handle, key.slice(), + &value.pinnable_slice())); + return value.Fetch(); + } catch (const ROCKSDB_NAMESPACE::KVException& e) { + return e.Code(); + } } -// TODO(AR) consider refactoring to share this between here and rocksjni.cc +// TODO(AR) consider refactoring to share this between here and forstjni.cc // used by txn_multi_get_helper below std::vector txn_column_families_helper( JNIEnv* env, jlongArray jcolumn_family_handles, bool* has_exception) { @@ -293,7 +317,7 @@ void free_key_values(std::vector& keys_to_free) { } } -// TODO(AR) consider refactoring to share this between here and rocksjni.cc +// TODO(AR) consider refactoring to share this between here and forstjni.cc // cf multi get jobjectArray txn_multi_get_helper(JNIEnv* env, const FnMultiGet& fn_multi_get, const jlong& jread_options_handle, @@ -379,11 +403,11 @@ jobjectArray txn_multi_get_helper(JNIEnv* env, const FnMultiGet& fn_multi_get, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: multiGet * Signature: (JJ[[B[J)[[B */ -jobjectArray Java_org_rocksdb_Transaction_multiGet__JJ_3_3B_3J( +jobjectArray Java_org_forstdb_Transaction_multiGet__JJ_3_3B_3J( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, jobjectArray jkey_parts, jlongArray jcolumn_family_handles) { bool has_exception = false; @@ -407,11 +431,11 @@ jobjectArray Java_org_rocksdb_Transaction_multiGet__JJ_3_3B_3J( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: multiGet * Signature: (JJ[[B)[[B */ -jobjectArray Java_org_rocksdb_Transaction_multiGet__JJ_3_3B( +jobjectArray Java_org_forstdb_Transaction_multiGet__JJ_3_3B( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, jobjectArray jkey_parts) { auto* txn = reinterpret_cast(jhandle); @@ -426,57 +450,101 @@ jobjectArray Java_org_rocksdb_Transaction_multiGet__JJ_3_3B( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getForUpdate - * Signature: (JJ[BIJZZ)[B + * Signature: (JJ[BIIJZZ)[B */ -jbyteArray Java_org_rocksdb_Transaction_getForUpdate__JJ_3BIJZZ( +jbyteArray Java_org_forstdb_Transaction_getForUpdate__JJ_3BIIJZZ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, - jbyteArray jkey, jint jkey_part_len, jlong jcolumn_family_handle, - jboolean jexclusive, jboolean jdo_validate) { + jbyteArray jkey, jint jkey_off, jint jkey_part_len, + jlong jcolumn_family_handle, jboolean jexclusive, jboolean jdo_validate) { + auto* read_options = + reinterpret_cast(jread_options_handle); auto* column_family_handle = reinterpret_cast( jcolumn_family_handle); auto* txn = reinterpret_cast(jhandle); - FnGet fn_get_for_update = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::GetForUpdate, txn, - std::placeholders::_1, column_family_handle, std::placeholders::_2, - std::placeholders::_3, jexclusive, jdo_validate); - return txn_get_helper(env, fn_get_for_update, jread_options_handle, jkey, - jkey_part_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArrayPinnableSlice value(env); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, + txn->GetForUpdate(*read_options, column_family_handle, key.slice(), + &value.pinnable_slice(), jexclusive, jdo_validate)); + return value.NewByteArray(); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return nullptr; + } } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getForUpdate - * Signature: (JJ[BIZZ)[B + * Signature: (JJ[BII[BIIJZZ)I */ -jbyteArray Java_org_rocksdb_Transaction_getForUpdate__JJ_3BIZZ( +jint Java_org_forstdb_Transaction_getForUpdate__JJ_3BII_3BIIJZZ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, - jbyteArray jkey, jint jkey_part_len, jboolean jexclusive, - jboolean jdo_validate) { + jbyteArray jkey, jint jkey_off, jint jkey_part_len, jbyteArray jval, + jint jval_off, jint jval_len, jlong jcolumn_family_handle, + jboolean jexclusive, jboolean jdo_validate) { auto* txn = reinterpret_cast(jhandle); - FnGet fn_get_for_update = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::GetForUpdate, txn, - std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, - jexclusive, jdo_validate); - return txn_get_helper(env, fn_get_for_update, jread_options_handle, jkey, - jkey_part_len); + auto* read_options = + reinterpret_cast(jread_options_handle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArrayPinnableSlice value(env, jval, jval_off, + jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, + txn->GetForUpdate(*read_options, column_family_handle, key.slice(), + &value.pinnable_slice(), jexclusive, jdo_validate)); + return value.Fetch(); + } catch (const ROCKSDB_NAMESPACE::KVException& e) { + return e.Code(); + } +} + +/* + * Class: org_forstdb_Transaction + * Method: getDirectForUpdate + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJZZ)I + */ +jint Java_org_forstdb_Transaction_getDirectForUpdate( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jobject jkey_bb, jint jkey_off, jint jkey_part_len, jobject jval_bb, + jint jval_off, jint jval_len, jlong jcolumn_family_handle, + jboolean jexclusive, jboolean jdo_validate) { + auto* txn = reinterpret_cast(jhandle); + auto* read_options = + reinterpret_cast(jread_options_handle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + + try { + ROCKSDB_NAMESPACE::JDirectBufferSlice key(env, jkey_bb, jkey_off, + jkey_part_len); + ROCKSDB_NAMESPACE::JDirectBufferPinnableSlice value(env, jval_bb, jval_off, + jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, + txn->GetForUpdate(*read_options, column_family_handle, key.slice(), + &value.pinnable_slice(), jexclusive, jdo_validate)); + return value.Fetch(); + } catch (const ROCKSDB_NAMESPACE::KVException& e) { + return e.Code(); + } } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: multiGetForUpdate * Signature: (JJ[[B[J)[[B */ -jobjectArray Java_org_rocksdb_Transaction_multiGetForUpdate__JJ_3_3B_3J( +jobjectArray Java_org_forstdb_Transaction_multiGetForUpdate__JJ_3_3B_3J( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, jobjectArray jkey_parts, jlongArray jcolumn_family_handles) { bool has_exception = false; @@ -501,11 +569,11 @@ jobjectArray Java_org_rocksdb_Transaction_multiGetForUpdate__JJ_3_3B_3J( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: multiGetForUpdate * Signature: (JJ[[B)[[B */ -jobjectArray Java_org_rocksdb_Transaction_multiGetForUpdate__JJ_3_3B( +jobjectArray Java_org_forstdb_Transaction_multiGetForUpdate__JJ_3_3B( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, jobjectArray jkey_parts) { auto* txn = reinterpret_cast(jhandle); @@ -520,28 +588,14 @@ jobjectArray Java_org_rocksdb_Transaction_multiGetForUpdate__JJ_3_3B( } /* - * Class: org_rocksdb_Transaction - * Method: getIterator - * Signature: (JJ)J - */ -jlong Java_org_rocksdb_Transaction_getIterator__JJ(JNIEnv* /*env*/, - jobject /*jobj*/, - jlong jhandle, - jlong jread_options_handle) { - auto* txn = reinterpret_cast(jhandle); - auto* read_options = - reinterpret_cast(jread_options_handle); - return GET_CPLUSPLUS_POINTER(txn->GetIterator(*read_options)); -} - -/* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getIterator * Signature: (JJJ)J */ -jlong Java_org_rocksdb_Transaction_getIterator__JJJ( - JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, - jlong jread_options_handle, jlong jcolumn_family_handle) { +jlong Java_org_forstdb_Transaction_getIterator(JNIEnv* /*env*/, + jobject /*jobj*/, jlong jhandle, + jlong jread_options_handle, + jlong jcolumn_family_handle) { auto* txn = reinterpret_cast(jhandle); auto* read_options = reinterpret_cast(jread_options_handle); @@ -552,91 +606,100 @@ jlong Java_org_rocksdb_Transaction_getIterator__JJJ( txn->GetIterator(*read_options, column_family_handle)); } -typedef std::function - FnWriteKV; - -// TODO(AR) consider refactoring to share this between here and rocksjni.cc -void txn_write_kv_helper(JNIEnv* env, const FnWriteKV& fn_write_kv, - const jbyteArray& jkey, const jint& jkey_part_len, - const jbyteArray& jval, const jint& jval_len) { - jbyte* key = env->GetByteArrayElements(jkey, nullptr); - if (key == nullptr) { - // exception thrown: OutOfMemoryError - return; - } - jbyte* value = env->GetByteArrayElements(jval, nullptr); - if (value == nullptr) { - // exception thrown: OutOfMemoryError - env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); +/* + * Class: org_forstdb_Transaction + * Method: put + * Signature: (J[BII[BIIJZ)V + */ +void Java_org_forstdb_Transaction_put__J_3BII_3BIIJZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_off, jint jkey_part_len, jbyteArray jval, jint jval_off, + jint jval_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Put(column_family_handle, key.slice(), value.slice(), + jassume_tracked)); + } catch (ROCKSDB_NAMESPACE::KVException&) { return; } - ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), - jkey_part_len); - ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), - jval_len); - - ROCKSDB_NAMESPACE::Status s = fn_write_kv(key_slice, value_slice); - - // trigger java unref on key. - // by passing JNI_ABORT, it will simply release the reference without - // copying the result back to the java byte array. - env->ReleaseByteArrayElements(jval, value, JNI_ABORT); - env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); +} - if (s.ok()) { +/* + * Class: org_forstdb_Transaction + * Method: put + * Signature: (J[BII[BII)V + */ +void Java_org_forstdb_Transaction_put__J_3BII_3BII( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_off, jint jkey_part_len, jbyteArray jval, jint jval_off, + jint jval_len) { + auto* txn = reinterpret_cast(jhandle); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Put(key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { return; } - ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); } /* - * Class: org_rocksdb_Transaction - * Method: put - * Signature: (J[BI[BIJZ)V + * Class: org_forstdb_Transaction + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJZ)V */ -void Java_org_rocksdb_Transaction_put__J_3BI_3BIJZ( - JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, - jint jkey_part_len, jbyteArray jval, jint jval_len, +void Java_org_forstdb_Transaction_putDirect__JLjava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2IIJZ( + JNIEnv* env, jobject, jlong jhandle, jobject jkey_bb, jint jkey_off, + jint jkey_len, jobject jval_bb, jint jval_off, jint jval_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { auto* txn = reinterpret_cast(jhandle); auto* column_family_handle = reinterpret_cast( jcolumn_family_handle); - FnWriteKV fn_put = - std::bind(&ROCKSDB_NAMESPACE::Transaction::Put, txn, - column_family_handle, std::placeholders::_1, - std::placeholders::_2, jassume_tracked); - txn_write_kv_helper(env, fn_put, jkey, jkey_part_len, jval, jval_len); + try { + ROCKSDB_NAMESPACE::JDirectBufferSlice key(env, jkey_bb, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JDirectBufferSlice value(env, jval_bb, jval_off, + jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Put(column_family_handle, key.slice(), value.slice(), + jassume_tracked)); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_Transaction - * Method: put - * Signature: (J[BI[BI)V + * Class: org_forstdb_Transaction + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)V */ -void Java_org_rocksdb_Transaction_put__J_3BI_3BI(JNIEnv* env, jobject /*jobj*/, - jlong jhandle, jbyteArray jkey, - jint jkey_part_len, - jbyteArray jval, - jint jval_len) { +void Java_org_forstdb_Transaction_putDirect__JLjava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2II( + JNIEnv* env, jobject, jlong jhandle, jobject jkey_bb, jint jkey_off, + jint jkey_len, jobject jval_bb, jint jval_off, jint jval_len) { auto* txn = reinterpret_cast(jhandle); - FnWriteKV fn_put = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::Put, txn, std::placeholders::_1, - std::placeholders::_2); - txn_write_kv_helper(env, fn_put, jkey, jkey_part_len, jval, jval_len); + try { + ROCKSDB_NAMESPACE::JDirectBufferSlice key(env, jkey_bb, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JDirectBufferSlice value(env, jval_bb, jval_off, + jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Put(key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } typedef std::function FnWriteKVParts; -// TODO(AR) consider refactoring to share this between here and rocksjni.cc +// TODO(AR) consider refactoring to share this between here and forstjni.cc void txn_write_kv_parts_helper(JNIEnv* env, const FnWriteKVParts& fn_write_kv_parts, const jobjectArray& jkey_parts, @@ -736,11 +799,11 @@ void txn_write_kv_parts_helper(JNIEnv* env, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: put * Signature: (J[[BI[[BIJZ)V */ -void Java_org_rocksdb_Transaction_put__J_3_3BI_3_3BIJZ( +void Java_org_forstdb_Transaction_put__J_3_3BI_3_3BIJZ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len, jobjectArray jvalue_parts, jint jvalue_parts_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { @@ -760,11 +823,11 @@ void Java_org_rocksdb_Transaction_put__J_3_3BI_3_3BIJZ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: put * Signature: (J[[BI[[BI)V */ -void Java_org_rocksdb_Transaction_put__J_3_3BI_3_3BI( +void Java_org_forstdb_Transaction_put__J_3_3BI_3_3BI( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len, jobjectArray jvalue_parts, jint jvalue_parts_len) { auto* txn = reinterpret_cast(jhandle); @@ -778,50 +841,101 @@ void Java_org_rocksdb_Transaction_put__J_3_3BI_3_3BI( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: merge - * Signature: (J[BI[BIJZ)V + * Signature: (J[BII[BIIJZ)V */ -void Java_org_rocksdb_Transaction_merge__J_3BI_3BIJZ( +void Java_org_forstdb_Transaction_merge__J_3BII_3BIIJZ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, - jint jkey_part_len, jbyteArray jval, jint jval_len, - jlong jcolumn_family_handle, jboolean jassume_tracked) { + jint jkey_off, jint jkey_part_len, jbyteArray jval, jint jval_off, + jint jval_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { auto* txn = reinterpret_cast(jhandle); auto* column_family_handle = reinterpret_cast( jcolumn_family_handle); - FnWriteKV fn_merge = - std::bind(&ROCKSDB_NAMESPACE::Transaction::Merge, txn, - column_family_handle, std::placeholders::_1, - std::placeholders::_2, jassume_tracked); - txn_write_kv_helper(env, fn_merge, jkey, jkey_part_len, jval, jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Merge(column_family_handle, key.slice(), value.slice(), + jassume_tracked)); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: merge - * Signature: (J[BI[BI)V + * Signature: (J[BII[BII)V */ -void Java_org_rocksdb_Transaction_merge__J_3BI_3BI( +void Java_org_forstdb_Transaction_merge__J_3BII_3BII( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, - jint jkey_part_len, jbyteArray jval, jint jval_len) { + jint jkey_off, jint jkey_part_len, jbyteArray jval, jint jval_off, + jint jval_len) { + auto* txn = reinterpret_cast(jhandle); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Merge(key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } +} + +/* + * Class: org_forstdb_Transaction + * Method: mergeDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJZ)V + */ +JNIEXPORT void JNICALL +Java_org_forstdb_Transaction_mergeDirect__JLjava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2IIJZ( + JNIEnv* env, jobject, jlong jhandle, jobject jkey_bb, jint jkey_off, + jint jkey_len, jobject jval_bb, jint jval_off, jint jval_len, + jlong jcolumn_family_handle, jboolean jassume_tracked) { auto* txn = reinterpret_cast(jhandle); - FnWriteKV fn_merge = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::Merge, txn, std::placeholders::_1, - std::placeholders::_2); - txn_write_kv_helper(env, fn_merge, jkey, jkey_part_len, jval, jval_len); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + try { + ROCKSDB_NAMESPACE::JDirectBufferSlice key(env, jkey_bb, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JDirectBufferSlice value(env, jval_bb, jval_off, + jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Merge(column_family_handle, key.slice(), value.slice(), + jassume_tracked)); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } +} + +/* + * Class: org_forstdb_Transaction + * Method: mergeDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL +Java_org_forstdb_Transaction_mergeDirect__JLjava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2II( + JNIEnv* env, jobject, jlong jhandle, jobject jkey_bb, jint jkey_off, + jint jkey_len, jobject jval_bb, jint jval_off, jint jval_len) { + auto* txn = reinterpret_cast(jhandle); + try { + ROCKSDB_NAMESPACE::JDirectBufferSlice key(env, jkey_bb, jkey_off, jkey_len); + ROCKSDB_NAMESPACE::JDirectBufferSlice value(env, jval_bb, jval_off, + jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->Merge(key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } typedef std::function FnWriteK; -// TODO(AR) consider refactoring to share this between here and rocksjni.cc +// TODO(AR) consider refactoring to share this between here and forstjni.cc void txn_write_k_helper(JNIEnv* env, const FnWriteK& fn_write_k, const jbyteArray& jkey, const jint& jkey_part_len) { jbyte* key = env->GetByteArrayElements(jkey, nullptr); @@ -846,11 +960,11 @@ void txn_write_k_helper(JNIEnv* env, const FnWriteK& fn_write_k, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: delete * Signature: (J[BIJZ)V */ -void Java_org_rocksdb_Transaction_delete__J_3BIJZ( +void Java_org_forstdb_Transaction_delete__J_3BIJZ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jint jkey_part_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { auto* txn = reinterpret_cast(jhandle); @@ -867,11 +981,11 @@ void Java_org_rocksdb_Transaction_delete__J_3BIJZ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: delete * Signature: (J[BI)V */ -void Java_org_rocksdb_Transaction_delete__J_3BI(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_Transaction_delete__J_3BI(JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jint jkey_part_len) { auto* txn = reinterpret_cast(jhandle); @@ -885,7 +999,7 @@ typedef std::function FnWriteKParts; -// TODO(AR) consider refactoring to share this between here and rocksjni.cc +// TODO(AR) consider refactoring to share this between here and forstjni.cc void txn_write_k_parts_helper(JNIEnv* env, const FnWriteKParts& fn_write_k_parts, const jobjectArray& jkey_parts, @@ -934,11 +1048,11 @@ void txn_write_k_parts_helper(JNIEnv* env, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: delete * Signature: (J[[BIJZ)V */ -void Java_org_rocksdb_Transaction_delete__J_3_3BIJZ( +void Java_org_forstdb_Transaction_delete__J_3_3BIJZ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { @@ -956,11 +1070,11 @@ void Java_org_rocksdb_Transaction_delete__J_3_3BIJZ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: delete * Signature: (J[[BI)V */ -void Java_org_rocksdb_Transaction_delete__J_3_3BI(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_Transaction_delete__J_3_3BI(JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len) { @@ -972,11 +1086,11 @@ void Java_org_rocksdb_Transaction_delete__J_3_3BI(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: singleDelete * Signature: (J[BIJZ)V */ -void Java_org_rocksdb_Transaction_singleDelete__J_3BIJZ( +void Java_org_forstdb_Transaction_singleDelete__J_3BIJZ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jint jkey_part_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { auto* txn = reinterpret_cast(jhandle); @@ -993,11 +1107,11 @@ void Java_org_rocksdb_Transaction_singleDelete__J_3BIJZ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: singleDelete * Signature: (J[BI)V */ -void Java_org_rocksdb_Transaction_singleDelete__J_3BI(JNIEnv* env, +void Java_org_forstdb_Transaction_singleDelete__J_3BI(JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, @@ -1011,11 +1125,11 @@ void Java_org_rocksdb_Transaction_singleDelete__J_3BI(JNIEnv* env, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: singleDelete * Signature: (J[[BIJZ)V */ -void Java_org_rocksdb_Transaction_singleDelete__J_3_3BIJZ( +void Java_org_forstdb_Transaction_singleDelete__J_3_3BIJZ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { @@ -1034,11 +1148,11 @@ void Java_org_rocksdb_Transaction_singleDelete__J_3_3BIJZ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: singleDelete * Signature: (J[[BI)V */ -void Java_org_rocksdb_Transaction_singleDelete__J_3_3BI(JNIEnv* env, +void Java_org_forstdb_Transaction_singleDelete__J_3_3BI(JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, @@ -1053,11 +1167,11 @@ void Java_org_rocksdb_Transaction_singleDelete__J_3_3BI(JNIEnv* env, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: putUntracked * Signature: (J[BI[BIJ)V */ -void Java_org_rocksdb_Transaction_putUntracked__J_3BI_3BIJ( +void Java_org_forstdb_Transaction_putUntracked__J_3BI_3BIJ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jint jkey_part_len, jbyteArray jval, jint jval_len, jlong jcolumn_family_handle) { @@ -1065,40 +1179,42 @@ void Java_org_rocksdb_Transaction_putUntracked__J_3BI_3BIJ( auto* column_family_handle = reinterpret_cast( jcolumn_family_handle); - FnWriteKV fn_put_untracked = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::PutUntracked, txn, - column_family_handle, std::placeholders::_1, std::placeholders::_2); - txn_write_kv_helper(env, fn_put_untracked, jkey, jkey_part_len, jval, - jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, 0, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, 0, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, + txn->PutUntracked(column_family_handle, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: putUntracked * Signature: (J[BI[BI)V */ -void Java_org_rocksdb_Transaction_putUntracked__J_3BI_3BI( +void Java_org_forstdb_Transaction_putUntracked__J_3BI_3BI( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jint jkey_part_len, jbyteArray jval, jint jval_len) { auto* txn = reinterpret_cast(jhandle); - FnWriteKV fn_put_untracked = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::PutUntracked, txn, - std::placeholders::_1, std::placeholders::_2); - txn_write_kv_helper(env, fn_put_untracked, jkey, jkey_part_len, jval, - jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, 0, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, 0, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, txn->PutUntracked(key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: putUntracked * Signature: (J[[BI[[BIJ)V */ -void Java_org_rocksdb_Transaction_putUntracked__J_3_3BI_3_3BIJ( +void Java_org_forstdb_Transaction_putUntracked__J_3_3BI_3_3BIJ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len, jobjectArray jvalue_parts, jint jvalue_parts_len, jlong jcolumn_family_handle) { @@ -1117,11 +1233,11 @@ void Java_org_rocksdb_Transaction_putUntracked__J_3_3BI_3_3BIJ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: putUntracked * Signature: (J[[BI[[BI)V */ -void Java_org_rocksdb_Transaction_putUntracked__J_3_3BI_3_3BI( +void Java_org_forstdb_Transaction_putUntracked__J_3_3BI_3_3BI( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len, jobjectArray jvalue_parts, jint jvalue_parts_len) { auto* txn = reinterpret_cast(jhandle); @@ -1135,52 +1251,60 @@ void Java_org_rocksdb_Transaction_putUntracked__J_3_3BI_3_3BI( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: mergeUntracked - * Signature: (J[BI[BIJ)V + * Signature: (J[BII[BIIJ)V */ -void Java_org_rocksdb_Transaction_mergeUntracked__J_3BI_3BIJ( +void Java_org_forstdb_Transaction_mergeUntracked( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, - jint jkey_part_len, jbyteArray jval, jint jval_len, - jlong jcolumn_family_handle) { + jint jkey_off, jint jkey_part_len, jbyteArray jval, jint jval_off, + jint jval_len, jlong jcolumn_family_handle) { auto* txn = reinterpret_cast(jhandle); auto* column_family_handle = reinterpret_cast( jcolumn_family_handle); - FnWriteKV fn_merge_untracked = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::MergeUntracked, txn, - column_family_handle, std::placeholders::_1, std::placeholders::_2); - txn_write_kv_helper(env, fn_merge_untracked, jkey, jkey_part_len, jval, - jval_len); + try { + ROCKSDB_NAMESPACE::JByteArraySlice key(env, jkey, jkey_off, jkey_part_len); + ROCKSDB_NAMESPACE::JByteArraySlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, + txn->MergeUntracked(column_family_handle, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_Transaction - * Method: mergeUntracked - * Signature: (J[BI[BI)V + * Class: org_forstdb_Transaction + * Method: mergeUntrackedDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V */ -void Java_org_rocksdb_Transaction_mergeUntracked__J_3BI_3BI( - JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, - jint jkey_part_len, jbyteArray jval, jint jval_len) { +void Java_org_forstdb_Transaction_mergeUntrackedDirect( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobject jkey, jint jkey_off, + jint jkey_part_len, jobject jval, jint jval_off, jint jval_len, + jlong jcolumn_family_handle) { auto* txn = reinterpret_cast(jhandle); - FnWriteKV fn_merge_untracked = - std::bind( - &ROCKSDB_NAMESPACE::Transaction::MergeUntracked, txn, - std::placeholders::_1, std::placeholders::_2); - txn_write_kv_helper(env, fn_merge_untracked, jkey, jkey_part_len, jval, - jval_len); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + try { + ROCKSDB_NAMESPACE::JDirectBufferSlice key(env, jkey, jkey_off, + jkey_part_len); + ROCKSDB_NAMESPACE::JDirectBufferSlice value(env, jval, jval_off, jval_len); + ROCKSDB_NAMESPACE::KVException::ThrowOnError( + env, + txn->MergeUntracked(column_family_handle, key.slice(), value.slice())); + } catch (ROCKSDB_NAMESPACE::KVException&) { + return; + } } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: deleteUntracked * Signature: (J[BIJ)V */ -void Java_org_rocksdb_Transaction_deleteUntracked__J_3BIJ( +void Java_org_forstdb_Transaction_deleteUntracked__J_3BIJ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jint jkey_part_len, jlong jcolumn_family_handle) { auto* txn = reinterpret_cast(jhandle); @@ -1196,11 +1320,11 @@ void Java_org_rocksdb_Transaction_deleteUntracked__J_3BIJ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: deleteUntracked * Signature: (J[BI)V */ -void Java_org_rocksdb_Transaction_deleteUntracked__J_3BI(JNIEnv* env, +void Java_org_forstdb_Transaction_deleteUntracked__J_3BI(JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, @@ -1214,11 +1338,11 @@ void Java_org_rocksdb_Transaction_deleteUntracked__J_3BI(JNIEnv* env, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: deleteUntracked * Signature: (J[[BIJ)V */ -void Java_org_rocksdb_Transaction_deleteUntracked__J_3_3BIJ( +void Java_org_forstdb_Transaction_deleteUntracked__J_3_3BIJ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len, jlong jcolumn_family_handle) { auto* txn = reinterpret_cast(jhandle); @@ -1236,11 +1360,11 @@ void Java_org_rocksdb_Transaction_deleteUntracked__J_3_3BIJ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: deleteUntracked * Signature: (J[[BI)V */ -void Java_org_rocksdb_Transaction_deleteUntracked__J_3_3BI( +void Java_org_forstdb_Transaction_deleteUntracked__J_3_3BI( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, jint jkey_parts_len) { auto* txn = reinterpret_cast(jhandle); @@ -1254,11 +1378,11 @@ void Java_org_rocksdb_Transaction_deleteUntracked__J_3_3BI( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: putLogData * Signature: (J[BI)V */ -void Java_org_rocksdb_Transaction_putLogData(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_Transaction_putLogData(JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jint jkey_part_len) { auto* txn = reinterpret_cast(jhandle); @@ -1280,11 +1404,11 @@ void Java_org_rocksdb_Transaction_putLogData(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: disableIndexing * Signature: (J)V */ -void Java_org_rocksdb_Transaction_disableIndexing(JNIEnv* /*env*/, +void Java_org_forstdb_Transaction_disableIndexing(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1292,11 +1416,11 @@ void Java_org_rocksdb_Transaction_disableIndexing(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: enableIndexing * Signature: (J)V */ -void Java_org_rocksdb_Transaction_enableIndexing(JNIEnv* /*env*/, +void Java_org_forstdb_Transaction_enableIndexing(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1304,33 +1428,33 @@ void Java_org_rocksdb_Transaction_enableIndexing(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getNumKeys * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getNumKeys(JNIEnv* /*env*/, jobject /*jobj*/, +jlong Java_org_forstdb_Transaction_getNumKeys(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); return txn->GetNumKeys(); } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getNumPuts * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getNumPuts(JNIEnv* /*env*/, jobject /*jobj*/, +jlong Java_org_forstdb_Transaction_getNumPuts(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); return txn->GetNumPuts(); } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getNumDeletes * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getNumDeletes(JNIEnv* /*env*/, +jlong Java_org_forstdb_Transaction_getNumDeletes(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1338,11 +1462,11 @@ jlong Java_org_rocksdb_Transaction_getNumDeletes(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getNumMerges * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getNumMerges(JNIEnv* /*env*/, +jlong Java_org_forstdb_Transaction_getNumMerges(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1350,11 +1474,11 @@ jlong Java_org_rocksdb_Transaction_getNumMerges(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getElapsedTime * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getElapsedTime(JNIEnv* /*env*/, +jlong Java_org_forstdb_Transaction_getElapsedTime(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1362,11 +1486,11 @@ jlong Java_org_rocksdb_Transaction_getElapsedTime(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getWriteBatch * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getWriteBatch(JNIEnv* /*env*/, +jlong Java_org_forstdb_Transaction_getWriteBatch(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1374,11 +1498,11 @@ jlong Java_org_rocksdb_Transaction_getWriteBatch(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: setLockTimeout * Signature: (JJ)V */ -void Java_org_rocksdb_Transaction_setLockTimeout(JNIEnv* /*env*/, +void Java_org_forstdb_Transaction_setLockTimeout(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jlock_timeout) { @@ -1387,11 +1511,11 @@ void Java_org_rocksdb_Transaction_setLockTimeout(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getWriteOptions * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getWriteOptions(JNIEnv* /*env*/, +jlong Java_org_forstdb_Transaction_getWriteOptions(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1399,11 +1523,11 @@ jlong Java_org_rocksdb_Transaction_getWriteOptions(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: setWriteOptions * Signature: (JJ)V */ -void Java_org_rocksdb_Transaction_setWriteOptions(JNIEnv* /*env*/, +void Java_org_forstdb_Transaction_setWriteOptions(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jwrite_options_handle) { @@ -1414,11 +1538,11 @@ void Java_org_rocksdb_Transaction_setWriteOptions(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: undo * Signature: (J[BIJ)V */ -void Java_org_rocksdb_Transaction_undoGetForUpdate__J_3BIJ( +void Java_org_forstdb_Transaction_undoGetForUpdate__J_3BIJ( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, jint jkey_part_len, jlong jcolumn_family_handle) { auto* txn = reinterpret_cast(jhandle); @@ -1439,11 +1563,11 @@ void Java_org_rocksdb_Transaction_undoGetForUpdate__J_3BIJ( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: undoGetForUpdate * Signature: (J[BI)V */ -void Java_org_rocksdb_Transaction_undoGetForUpdate__J_3BI(JNIEnv* env, +void Java_org_forstdb_Transaction_undoGetForUpdate__J_3BI(JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, @@ -1463,11 +1587,11 @@ void Java_org_rocksdb_Transaction_undoGetForUpdate__J_3BI(JNIEnv* env, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: rebuildFromWriteBatch * Signature: (JJ)V */ -void Java_org_rocksdb_Transaction_rebuildFromWriteBatch( +void Java_org_forstdb_Transaction_rebuildFromWriteBatch( JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jwrite_batch_handle) { auto* txn = reinterpret_cast(jhandle); auto* write_batch = @@ -1479,11 +1603,11 @@ void Java_org_rocksdb_Transaction_rebuildFromWriteBatch( } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getCommitTimeWriteBatch * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getCommitTimeWriteBatch(JNIEnv* /*env*/, +jlong Java_org_forstdb_Transaction_getCommitTimeWriteBatch(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1491,11 +1615,11 @@ jlong Java_org_rocksdb_Transaction_getCommitTimeWriteBatch(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: setLogNumber * Signature: (JJ)V */ -void Java_org_rocksdb_Transaction_setLogNumber(JNIEnv* /*env*/, +void Java_org_forstdb_Transaction_setLogNumber(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jlog_number) { auto* txn = reinterpret_cast(jhandle); @@ -1503,11 +1627,11 @@ void Java_org_rocksdb_Transaction_setLogNumber(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getLogNumber * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getLogNumber(JNIEnv* /*env*/, +jlong Java_org_forstdb_Transaction_getLogNumber(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1515,11 +1639,11 @@ jlong Java_org_rocksdb_Transaction_getLogNumber(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: setName * Signature: (JLjava/lang/String;)V */ -void Java_org_rocksdb_Transaction_setName(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_Transaction_setName(JNIEnv* env, jobject /*jobj*/, jlong jhandle, jstring jname) { auto* txn = reinterpret_cast(jhandle); const char* name = env->GetStringUTFChars(jname, nullptr); @@ -1538,11 +1662,11 @@ void Java_org_rocksdb_Transaction_setName(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getName * Signature: (J)Ljava/lang/String; */ -jstring Java_org_rocksdb_Transaction_getName(JNIEnv* env, jobject /*jobj*/, +jstring Java_org_forstdb_Transaction_getName(JNIEnv* env, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); ROCKSDB_NAMESPACE::TransactionName name = txn->GetName(); @@ -1550,11 +1674,11 @@ jstring Java_org_rocksdb_Transaction_getName(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getID * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getID(JNIEnv* /*env*/, jobject /*jobj*/, +jlong Java_org_forstdb_Transaction_getID(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); ROCKSDB_NAMESPACE::TransactionID id = txn->GetID(); @@ -1562,11 +1686,11 @@ jlong Java_org_rocksdb_Transaction_getID(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: isDeadlockDetect * Signature: (J)Z */ -jboolean Java_org_rocksdb_Transaction_isDeadlockDetect(JNIEnv* /*env*/, +jboolean Java_org_forstdb_Transaction_isDeadlockDetect(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1574,11 +1698,11 @@ jboolean Java_org_rocksdb_Transaction_isDeadlockDetect(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getWaitingTxns - * Signature: (J)Lorg/rocksdb/Transaction/WaitingTransactions; + * Signature: (J)Lorg/forstdb/Transaction/WaitingTransactions; */ -jobject Java_org_rocksdb_Transaction_getWaitingTxns(JNIEnv* env, +jobject Java_org_forstdb_Transaction_getWaitingTxns(JNIEnv* env, jobject jtransaction_obj, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); @@ -1593,11 +1717,11 @@ jobject Java_org_rocksdb_Transaction_getWaitingTxns(JNIEnv* env, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getState * Signature: (J)B */ -jbyte Java_org_rocksdb_Transaction_getState(JNIEnv* /*env*/, jobject /*jobj*/, +jbyte Java_org_forstdb_Transaction_getState(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); ROCKSDB_NAMESPACE::Transaction::TransactionState txn_status = txn->GetState(); @@ -1632,11 +1756,11 @@ jbyte Java_org_rocksdb_Transaction_getState(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: getId * Signature: (J)J */ -jlong Java_org_rocksdb_Transaction_getId(JNIEnv* /*env*/, jobject /*jobj*/, +jlong Java_org_forstdb_Transaction_getId(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* txn = reinterpret_cast(jhandle); uint64_t id = txn->GetId(); @@ -1644,11 +1768,11 @@ jlong Java_org_rocksdb_Transaction_getId(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_Transaction + * Class: org_forstdb_Transaction * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_Transaction_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_Transaction_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { delete reinterpret_cast(jhandle); diff --git a/java/rocksjni/transaction_db.cc b/java/forstjni/transaction_db.cc similarity index 90% rename from java/rocksjni/transaction_db.cc rename to java/forstjni/transaction_db.cc index 0adf85606..e7df93ebd 100644 --- a/java/rocksjni/transaction_db.cc +++ b/java/forstjni/transaction_db.cc @@ -14,18 +14,18 @@ #include #include -#include "include/org_rocksdb_TransactionDB.h" +#include "include/org_forstdb_TransactionDB.h" #include "rocksdb/options.h" #include "rocksdb/utilities/transaction.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: open * Signature: (JJLjava/lang/String;)J */ -jlong Java_org_rocksdb_TransactionDB_open__JJLjava_lang_String_2( +jlong Java_org_forstdb_TransactionDB_open__JJLjava_lang_String_2( JNIEnv* env, jclass, jlong joptions_handle, jlong jtxn_db_options_handle, jstring jdb_path) { auto* options = @@ -52,11 +52,11 @@ jlong Java_org_rocksdb_TransactionDB_open__JJLjava_lang_String_2( } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: open * Signature: (JJLjava/lang/String;[[B[J)[J */ -jlongArray Java_org_rocksdb_TransactionDB_open__JJLjava_lang_String_2_3_3B_3J( +jlongArray Java_org_forstdb_TransactionDB_open__JJLjava_lang_String_2_3_3B_3J( JNIEnv* env, jclass, jlong jdb_options_handle, jlong jtxn_db_options_handle, jstring jdb_path, jobjectArray jcolumn_names, jlongArray jcolumn_options_handles) { @@ -143,11 +143,11 @@ jlongArray Java_org_rocksdb_TransactionDB_open__JJLjava_lang_String_2_3_3B_3J( } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_TransactionDB_disposeInternal(JNIEnv*, jobject, +void Java_org_forstdb_TransactionDB_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* txn_db = reinterpret_cast(jhandle); assert(txn_db != nullptr); @@ -155,11 +155,11 @@ void Java_org_rocksdb_TransactionDB_disposeInternal(JNIEnv*, jobject, } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: closeDatabase * Signature: (J)V */ -void Java_org_rocksdb_TransactionDB_closeDatabase(JNIEnv* env, jclass, +void Java_org_forstdb_TransactionDB_closeDatabase(JNIEnv* env, jclass, jlong jhandle) { auto* txn_db = reinterpret_cast(jhandle); assert(txn_db != nullptr); @@ -168,11 +168,11 @@ void Java_org_rocksdb_TransactionDB_closeDatabase(JNIEnv* env, jclass, } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: beginTransaction * Signature: (JJ)J */ -jlong Java_org_rocksdb_TransactionDB_beginTransaction__JJ( +jlong Java_org_forstdb_TransactionDB_beginTransaction__JJ( JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle) { auto* txn_db = reinterpret_cast(jhandle); auto* write_options = @@ -183,11 +183,11 @@ jlong Java_org_rocksdb_TransactionDB_beginTransaction__JJ( } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: beginTransaction * Signature: (JJJ)J */ -jlong Java_org_rocksdb_TransactionDB_beginTransaction__JJJ( +jlong Java_org_forstdb_TransactionDB_beginTransaction__JJJ( JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, jlong jtxn_options_handle) { auto* txn_db = reinterpret_cast(jhandle); @@ -201,11 +201,11 @@ jlong Java_org_rocksdb_TransactionDB_beginTransaction__JJJ( } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: beginTransaction_withOld * Signature: (JJJ)J */ -jlong Java_org_rocksdb_TransactionDB_beginTransaction_1withOld__JJJ( +jlong Java_org_forstdb_TransactionDB_beginTransaction_1withOld__JJJ( JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, jlong jold_txn_handle) { auto* txn_db = reinterpret_cast(jhandle); @@ -226,11 +226,11 @@ jlong Java_org_rocksdb_TransactionDB_beginTransaction_1withOld__JJJ( } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: beginTransaction_withOld * Signature: (JJJJ)J */ -jlong Java_org_rocksdb_TransactionDB_beginTransaction_1withOld__JJJJ( +jlong Java_org_forstdb_TransactionDB_beginTransaction_1withOld__JJJJ( JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, jlong jtxn_options_handle, jlong jold_txn_handle) { auto* txn_db = reinterpret_cast(jhandle); @@ -252,11 +252,11 @@ jlong Java_org_rocksdb_TransactionDB_beginTransaction_1withOld__JJJJ( } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: getTransactionByName * Signature: (JLjava/lang/String;)J */ -jlong Java_org_rocksdb_TransactionDB_getTransactionByName(JNIEnv* env, jobject, +jlong Java_org_forstdb_TransactionDB_getTransactionByName(JNIEnv* env, jobject, jlong jhandle, jstring jname) { auto* txn_db = reinterpret_cast(jhandle); @@ -271,11 +271,11 @@ jlong Java_org_rocksdb_TransactionDB_getTransactionByName(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: getAllPreparedTransactions * Signature: (J)[J */ -jlongArray Java_org_rocksdb_TransactionDB_getAllPreparedTransactions( +jlongArray Java_org_forstdb_TransactionDB_getAllPreparedTransactions( JNIEnv* env, jobject, jlong jhandle) { auto* txn_db = reinterpret_cast(jhandle); std::vector txns; @@ -306,11 +306,11 @@ jlongArray Java_org_rocksdb_TransactionDB_getAllPreparedTransactions( } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: getLockStatusData * Signature: (J)Ljava/util/Map; */ -jobject Java_org_rocksdb_TransactionDB_getLockStatusData(JNIEnv* env, jobject, +jobject Java_org_forstdb_TransactionDB_getLockStatusData(JNIEnv* env, jobject, jlong jhandle) { auto* txn_db = reinterpret_cast(jhandle); const std::unordered_multimap @@ -355,11 +355,11 @@ jobject Java_org_rocksdb_TransactionDB_getLockStatusData(JNIEnv* env, jobject, } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: getDeadlockInfoBuffer - * Signature: (J)[Lorg/rocksdb/TransactionDB/DeadlockPath; + * Signature: (J)[Lorg/forstdb/TransactionDB/DeadlockPath; */ -jobjectArray Java_org_rocksdb_TransactionDB_getDeadlockInfoBuffer( +jobjectArray Java_org_forstdb_TransactionDB_getDeadlockInfoBuffer( JNIEnv* env, jobject jobj, jlong jhandle) { auto* txn_db = reinterpret_cast(jhandle); const std::vector deadlock_info_buffer = @@ -440,11 +440,11 @@ jobjectArray Java_org_rocksdb_TransactionDB_getDeadlockInfoBuffer( } /* - * Class: org_rocksdb_TransactionDB + * Class: org_forstdb_TransactionDB * Method: setDeadlockInfoBufferSize * Signature: (JI)V */ -void Java_org_rocksdb_TransactionDB_setDeadlockInfoBufferSize( +void Java_org_forstdb_TransactionDB_setDeadlockInfoBufferSize( JNIEnv*, jobject, jlong jhandle, jint jdeadlock_info_buffer_size) { auto* txn_db = reinterpret_cast(jhandle); txn_db->SetDeadlockInfoBufferSize(jdeadlock_info_buffer_size); diff --git a/java/rocksjni/transaction_db_options.cc b/java/forstjni/transaction_db_options.cc similarity index 75% rename from java/rocksjni/transaction_db_options.cc rename to java/forstjni/transaction_db_options.cc index 4cf27121e..d908ad37d 100644 --- a/java/rocksjni/transaction_db_options.cc +++ b/java/forstjni/transaction_db_options.cc @@ -8,17 +8,17 @@ #include -#include "include/org_rocksdb_TransactionDBOptions.h" +#include "include/org_forstdb_TransactionDBOptions.h" #include "rocksdb/utilities/transaction_db.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: newTransactionDBOptions * Signature: ()J */ -jlong Java_org_rocksdb_TransactionDBOptions_newTransactionDBOptions( +jlong Java_org_forstdb_TransactionDBOptions_newTransactionDBOptions( JNIEnv* /*env*/, jclass /*jcls*/) { ROCKSDB_NAMESPACE::TransactionDBOptions* opts = new ROCKSDB_NAMESPACE::TransactionDBOptions(); @@ -26,11 +26,11 @@ jlong Java_org_rocksdb_TransactionDBOptions_newTransactionDBOptions( } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: getMaxNumLocks * Signature: (J)J */ -jlong Java_org_rocksdb_TransactionDBOptions_getMaxNumLocks(JNIEnv* /*env*/, +jlong Java_org_forstdb_TransactionDBOptions_getMaxNumLocks(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = @@ -39,11 +39,11 @@ jlong Java_org_rocksdb_TransactionDBOptions_getMaxNumLocks(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: setMaxNumLocks * Signature: (JJ)V */ -void Java_org_rocksdb_TransactionDBOptions_setMaxNumLocks( +void Java_org_forstdb_TransactionDBOptions_setMaxNumLocks( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jmax_num_locks) { auto* opts = reinterpret_cast(jhandle); @@ -51,11 +51,11 @@ void Java_org_rocksdb_TransactionDBOptions_setMaxNumLocks( } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: getNumStripes * Signature: (J)J */ -jlong Java_org_rocksdb_TransactionDBOptions_getNumStripes(JNIEnv* /*env*/, +jlong Java_org_forstdb_TransactionDBOptions_getNumStripes(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = @@ -64,11 +64,11 @@ jlong Java_org_rocksdb_TransactionDBOptions_getNumStripes(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: setNumStripes * Signature: (JJ)V */ -void Java_org_rocksdb_TransactionDBOptions_setNumStripes(JNIEnv* /*env*/, +void Java_org_forstdb_TransactionDBOptions_setNumStripes(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jnum_stripes) { @@ -78,11 +78,11 @@ void Java_org_rocksdb_TransactionDBOptions_setNumStripes(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: getTransactionLockTimeout * Signature: (J)J */ -jlong Java_org_rocksdb_TransactionDBOptions_getTransactionLockTimeout( +jlong Java_org_forstdb_TransactionDBOptions_getTransactionLockTimeout( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -90,11 +90,11 @@ jlong Java_org_rocksdb_TransactionDBOptions_getTransactionLockTimeout( } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: setTransactionLockTimeout * Signature: (JJ)V */ -void Java_org_rocksdb_TransactionDBOptions_setTransactionLockTimeout( +void Java_org_forstdb_TransactionDBOptions_setTransactionLockTimeout( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jtransaction_lock_timeout) { auto* opts = @@ -103,11 +103,11 @@ void Java_org_rocksdb_TransactionDBOptions_setTransactionLockTimeout( } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: getDefaultLockTimeout * Signature: (J)J */ -jlong Java_org_rocksdb_TransactionDBOptions_getDefaultLockTimeout( +jlong Java_org_forstdb_TransactionDBOptions_getDefaultLockTimeout( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -115,11 +115,11 @@ jlong Java_org_rocksdb_TransactionDBOptions_getDefaultLockTimeout( } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: setDefaultLockTimeout * Signature: (JJ)V */ -void Java_org_rocksdb_TransactionDBOptions_setDefaultLockTimeout( +void Java_org_forstdb_TransactionDBOptions_setDefaultLockTimeout( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jdefault_lock_timeout) { auto* opts = @@ -128,11 +128,11 @@ void Java_org_rocksdb_TransactionDBOptions_setDefaultLockTimeout( } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: getWritePolicy * Signature: (J)B */ -jbyte Java_org_rocksdb_TransactionDBOptions_getWritePolicy(JNIEnv* /*env*/, +jbyte Java_org_forstdb_TransactionDBOptions_getWritePolicy(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = @@ -142,11 +142,11 @@ jbyte Java_org_rocksdb_TransactionDBOptions_getWritePolicy(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: setWritePolicy * Signature: (JB)V */ -void Java_org_rocksdb_TransactionDBOptions_setWritePolicy(JNIEnv* /*env*/, +void Java_org_forstdb_TransactionDBOptions_setWritePolicy(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jbyte jwrite_policy) { @@ -158,11 +158,11 @@ void Java_org_rocksdb_TransactionDBOptions_setWritePolicy(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionDBOptions + * Class: org_forstdb_TransactionDBOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_TransactionDBOptions_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_TransactionDBOptions_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { delete reinterpret_cast(jhandle); diff --git a/java/rocksjni/transaction_log.cc b/java/forstjni/transaction_log.cc similarity index 75% rename from java/rocksjni/transaction_log.cc rename to java/forstjni/transaction_log.cc index 97c3bb301..1a4719d75 100644 --- a/java/rocksjni/transaction_log.cc +++ b/java/forstjni/transaction_log.cc @@ -12,26 +12,26 @@ #include #include -#include "include/org_rocksdb_TransactionLogIterator.h" -#include "rocksjni/portal.h" +#include "include/org_forstdb_TransactionLogIterator.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_TransactionLogIterator + * Class: org_forstdb_TransactionLogIterator * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_TransactionLogIterator_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_TransactionLogIterator_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { delete reinterpret_cast(handle); } /* - * Class: org_rocksdb_TransactionLogIterator + * Class: org_forstdb_TransactionLogIterator * Method: isValid * Signature: (J)Z */ -jboolean Java_org_rocksdb_TransactionLogIterator_isValid(JNIEnv* /*env*/, +jboolean Java_org_forstdb_TransactionLogIterator_isValid(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { return reinterpret_cast(handle) @@ -39,22 +39,22 @@ jboolean Java_org_rocksdb_TransactionLogIterator_isValid(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionLogIterator + * Class: org_forstdb_TransactionLogIterator * Method: next * Signature: (J)V */ -void Java_org_rocksdb_TransactionLogIterator_next(JNIEnv* /*env*/, +void Java_org_forstdb_TransactionLogIterator_next(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->Next(); } /* - * Class: org_rocksdb_TransactionLogIterator + * Class: org_forstdb_TransactionLogIterator * Method: status * Signature: (J)V */ -void Java_org_rocksdb_TransactionLogIterator_status(JNIEnv* env, +void Java_org_forstdb_TransactionLogIterator_status(JNIEnv* env, jobject /*jobj*/, jlong handle) { ROCKSDB_NAMESPACE::Status s = @@ -66,11 +66,11 @@ void Java_org_rocksdb_TransactionLogIterator_status(JNIEnv* env, } /* - * Class: org_rocksdb_TransactionLogIterator + * Class: org_forstdb_TransactionLogIterator * Method: getBatch - * Signature: (J)Lorg/rocksdb/TransactionLogIterator$BatchResult + * Signature: (J)Lorg/forstdb/TransactionLogIterator$BatchResult */ -jobject Java_org_rocksdb_TransactionLogIterator_getBatch(JNIEnv* env, +jobject Java_org_forstdb_TransactionLogIterator_getBatch(JNIEnv* env, jobject /*jobj*/, jlong handle) { ROCKSDB_NAMESPACE::BatchResult batch_result = diff --git a/java/rocksjni/transaction_notifier.cc b/java/forstjni/transaction_notifier.cc similarity index 76% rename from java/rocksjni/transaction_notifier.cc rename to java/forstjni/transaction_notifier.cc index cefeb648a..1a556460c 100644 --- a/java/rocksjni/transaction_notifier.cc +++ b/java/forstjni/transaction_notifier.cc @@ -8,16 +8,16 @@ #include -#include "include/org_rocksdb_AbstractTransactionNotifier.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/transaction_notifier_jnicallback.h" +#include "include/org_forstdb_AbstractTransactionNotifier.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/transaction_notifier_jnicallback.h" /* - * Class: org_rocksdb_AbstractTransactionNotifier + * Class: org_forstdb_AbstractTransactionNotifier * Method: createNewTransactionNotifier * Signature: ()J */ -jlong Java_org_rocksdb_AbstractTransactionNotifier_createNewTransactionNotifier( +jlong Java_org_forstdb_AbstractTransactionNotifier_createNewTransactionNotifier( JNIEnv* env, jobject jobj) { auto* transaction_notifier = new ROCKSDB_NAMESPACE::TransactionNotifierJniCallback(env, jobj); @@ -28,11 +28,11 @@ jlong Java_org_rocksdb_AbstractTransactionNotifier_createNewTransactionNotifier( } /* - * Class: org_rocksdb_AbstractTransactionNotifier + * Class: org_forstdb_AbstractTransactionNotifier * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_AbstractTransactionNotifier_disposeInternal( +void Java_org_forstdb_AbstractTransactionNotifier_disposeInternal( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { // TODO(AR) refactor to use JniCallback::JniCallback // when https://github.com/facebook/rocksdb/pull/1241/ is merged diff --git a/java/rocksjni/transaction_notifier_jnicallback.cc b/java/forstjni/transaction_notifier_jnicallback.cc similarity index 90% rename from java/rocksjni/transaction_notifier_jnicallback.cc rename to java/forstjni/transaction_notifier_jnicallback.cc index 26761cabd..abd133e3b 100644 --- a/java/rocksjni/transaction_notifier_jnicallback.cc +++ b/java/forstjni/transaction_notifier_jnicallback.cc @@ -6,10 +6,10 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::TransactionNotifier. -#include "rocksjni/transaction_notifier_jnicallback.h" +#include "forstjni/transaction_notifier_jnicallback.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/transaction_notifier_jnicallback.h b/java/forstjni/transaction_notifier_jnicallback.h similarity index 97% rename from java/rocksjni/transaction_notifier_jnicallback.h rename to java/forstjni/transaction_notifier_jnicallback.h index 089a5ee4a..d31c7b22e 100644 --- a/java/rocksjni/transaction_notifier_jnicallback.h +++ b/java/forstjni/transaction_notifier_jnicallback.h @@ -12,7 +12,7 @@ #include #include "rocksdb/utilities/transaction.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/transaction_options.cc b/java/forstjni/transaction_options.cc similarity index 75% rename from java/rocksjni/transaction_options.cc rename to java/forstjni/transaction_options.cc index dcf363e14..8cf3339c9 100644 --- a/java/rocksjni/transaction_options.cc +++ b/java/forstjni/transaction_options.cc @@ -8,27 +8,27 @@ #include -#include "include/org_rocksdb_TransactionOptions.h" +#include "include/org_forstdb_TransactionOptions.h" #include "rocksdb/utilities/transaction_db.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: newTransactionOptions * Signature: ()J */ -jlong Java_org_rocksdb_TransactionOptions_newTransactionOptions( +jlong Java_org_forstdb_TransactionOptions_newTransactionOptions( JNIEnv* /*env*/, jclass /*jcls*/) { auto* opts = new ROCKSDB_NAMESPACE::TransactionOptions(); return GET_CPLUSPLUS_POINTER(opts); } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: isSetSnapshot * Signature: (J)Z */ -jboolean Java_org_rocksdb_TransactionOptions_isSetSnapshot(JNIEnv* /*env*/, +jboolean Java_org_forstdb_TransactionOptions_isSetSnapshot(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = @@ -37,11 +37,11 @@ jboolean Java_org_rocksdb_TransactionOptions_isSetSnapshot(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: setSetSnapshot * Signature: (JZ)V */ -void Java_org_rocksdb_TransactionOptions_setSetSnapshot( +void Java_org_forstdb_TransactionOptions_setSetSnapshot( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean jset_snapshot) { auto* opts = reinterpret_cast(jhandle); @@ -49,11 +49,11 @@ void Java_org_rocksdb_TransactionOptions_setSetSnapshot( } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: isDeadlockDetect * Signature: (J)Z */ -jboolean Java_org_rocksdb_TransactionOptions_isDeadlockDetect(JNIEnv* /*env*/, +jboolean Java_org_forstdb_TransactionOptions_isDeadlockDetect(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = @@ -62,11 +62,11 @@ jboolean Java_org_rocksdb_TransactionOptions_isDeadlockDetect(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: setDeadlockDetect * Signature: (JZ)V */ -void Java_org_rocksdb_TransactionOptions_setDeadlockDetect( +void Java_org_forstdb_TransactionOptions_setDeadlockDetect( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean jdeadlock_detect) { auto* opts = @@ -75,11 +75,11 @@ void Java_org_rocksdb_TransactionOptions_setDeadlockDetect( } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: getLockTimeout * Signature: (J)J */ -jlong Java_org_rocksdb_TransactionOptions_getLockTimeout(JNIEnv* /*env*/, +jlong Java_org_forstdb_TransactionOptions_getLockTimeout(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = @@ -88,11 +88,11 @@ jlong Java_org_rocksdb_TransactionOptions_getLockTimeout(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: setLockTimeout * Signature: (JJ)V */ -void Java_org_rocksdb_TransactionOptions_setLockTimeout(JNIEnv* /*env*/, +void Java_org_forstdb_TransactionOptions_setLockTimeout(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jlock_timeout) { @@ -102,11 +102,11 @@ void Java_org_rocksdb_TransactionOptions_setLockTimeout(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: getExpiration * Signature: (J)J */ -jlong Java_org_rocksdb_TransactionOptions_getExpiration(JNIEnv* /*env*/, +jlong Java_org_forstdb_TransactionOptions_getExpiration(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = @@ -115,11 +115,11 @@ jlong Java_org_rocksdb_TransactionOptions_getExpiration(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: setExpiration * Signature: (JJ)V */ -void Java_org_rocksdb_TransactionOptions_setExpiration(JNIEnv* /*env*/, +void Java_org_forstdb_TransactionOptions_setExpiration(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jexpiration) { @@ -129,11 +129,11 @@ void Java_org_rocksdb_TransactionOptions_setExpiration(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: getDeadlockDetectDepth * Signature: (J)J */ -jlong Java_org_rocksdb_TransactionOptions_getDeadlockDetectDepth( +jlong Java_org_forstdb_TransactionOptions_getDeadlockDetectDepth( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = reinterpret_cast(jhandle); @@ -141,11 +141,11 @@ jlong Java_org_rocksdb_TransactionOptions_getDeadlockDetectDepth( } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: setDeadlockDetectDepth * Signature: (JJ)V */ -void Java_org_rocksdb_TransactionOptions_setDeadlockDetectDepth( +void Java_org_forstdb_TransactionOptions_setDeadlockDetectDepth( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jdeadlock_detect_depth) { auto* opts = @@ -154,11 +154,11 @@ void Java_org_rocksdb_TransactionOptions_setDeadlockDetectDepth( } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: getMaxWriteBatchSize * Signature: (J)J */ -jlong Java_org_rocksdb_TransactionOptions_getMaxWriteBatchSize(JNIEnv* /*env*/, +jlong Java_org_forstdb_TransactionOptions_getMaxWriteBatchSize(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* opts = @@ -167,11 +167,11 @@ jlong Java_org_rocksdb_TransactionOptions_getMaxWriteBatchSize(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: setMaxWriteBatchSize * Signature: (JJ)V */ -void Java_org_rocksdb_TransactionOptions_setMaxWriteBatchSize( +void Java_org_forstdb_TransactionOptions_setMaxWriteBatchSize( JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jmax_write_batch_size) { auto* opts = @@ -180,11 +180,11 @@ void Java_org_rocksdb_TransactionOptions_setMaxWriteBatchSize( } /* - * Class: org_rocksdb_TransactionOptions + * Class: org_forstdb_TransactionOptions * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_TransactionOptions_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_TransactionOptions_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { delete reinterpret_cast(jhandle); diff --git a/java/rocksjni/ttl.cc b/java/forstjni/ttl.cc similarity index 91% rename from java/rocksjni/ttl.cc rename to java/forstjni/ttl.cc index 1fe2083d9..98cd83b20 100644 --- a/java/rocksjni/ttl.cc +++ b/java/forstjni/ttl.cc @@ -15,17 +15,17 @@ #include #include -#include "include/org_rocksdb_TtlDB.h" +#include "include/org_forstdb_TtlDB.h" #include "rocksdb/utilities/db_ttl.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_TtlDB + * Class: org_forstdb_TtlDB * Method: open * Signature: (JLjava/lang/String;IZ)J */ -jlong Java_org_rocksdb_TtlDB_open(JNIEnv* env, jclass, jlong joptions_handle, +jlong Java_org_forstdb_TtlDB_open(JNIEnv* env, jclass, jlong joptions_handle, jstring jdb_path, jint jttl, jboolean jread_only) { const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); @@ -51,11 +51,11 @@ jlong Java_org_rocksdb_TtlDB_open(JNIEnv* env, jclass, jlong joptions_handle, } /* - * Class: org_rocksdb_TtlDB + * Class: org_forstdb_TtlDB * Method: openCF * Signature: (JLjava/lang/String;[[B[J[IZ)[J */ -jlongArray Java_org_rocksdb_TtlDB_openCF(JNIEnv* env, jclass, jlong jopt_handle, +jlongArray Java_org_forstdb_TtlDB_openCF(JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, jobjectArray jcolumn_names, jlongArray jcolumn_options, @@ -150,22 +150,22 @@ jlongArray Java_org_rocksdb_TtlDB_openCF(JNIEnv* env, jclass, jlong jopt_handle, } /* - * Class: org_rocksdb_TtlDB + * Class: org_forstdb_TtlDB * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_TtlDB_disposeInternal(JNIEnv*, jobject, jlong jhandle) { +void Java_org_forstdb_TtlDB_disposeInternal(JNIEnv*, jobject, jlong jhandle) { auto* ttl_db = reinterpret_cast(jhandle); assert(ttl_db != nullptr); delete ttl_db; } /* - * Class: org_rocksdb_TtlDB + * Class: org_forstdb_TtlDB * Method: closeDatabase * Signature: (J)V */ -void Java_org_rocksdb_TtlDB_closeDatabase(JNIEnv* /* env */, jclass, +void Java_org_forstdb_TtlDB_closeDatabase(JNIEnv* /* env */, jclass, jlong /* jhandle */) { // auto* ttl_db = reinterpret_cast(jhandle); // assert(ttl_db != nullptr); @@ -177,11 +177,11 @@ void Java_org_rocksdb_TtlDB_closeDatabase(JNIEnv* /* env */, jclass, } /* - * Class: org_rocksdb_TtlDB + * Class: org_forstdb_TtlDB * Method: createColumnFamilyWithTtl - * Signature: (JLorg/rocksdb/ColumnFamilyDescriptor;[BJI)J; + * Signature: (JLorg/forstdb/ColumnFamilyDescriptor;[BJI)J; */ -jlong Java_org_rocksdb_TtlDB_createColumnFamilyWithTtl(JNIEnv* env, jobject, +jlong Java_org_forstdb_TtlDB_createColumnFamilyWithTtl(JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jcolumn_name, jlong jcolumn_options, diff --git a/java/rocksjni/wal_filter.cc b/java/forstjni/wal_filter.cc similarity index 71% rename from java/rocksjni/wal_filter.cc rename to java/forstjni/wal_filter.cc index 24b88afed..3fbd59a7a 100644 --- a/java/rocksjni/wal_filter.cc +++ b/java/forstjni/wal_filter.cc @@ -8,16 +8,16 @@ #include -#include "include/org_rocksdb_AbstractWalFilter.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/wal_filter_jnicallback.h" +#include "include/org_forstdb_AbstractWalFilter.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/wal_filter_jnicallback.h" /* - * Class: org_rocksdb_AbstractWalFilter + * Class: org_forstdb_AbstractWalFilter * Method: createNewWalFilter * Signature: ()J */ -jlong Java_org_rocksdb_AbstractWalFilter_createNewWalFilter(JNIEnv* env, +jlong Java_org_forstdb_AbstractWalFilter_createNewWalFilter(JNIEnv* env, jobject jobj) { auto* wal_filter = new ROCKSDB_NAMESPACE::WalFilterJniCallback(env, jobj); return GET_CPLUSPLUS_POINTER(wal_filter); diff --git a/java/rocksjni/wal_filter_jnicallback.cc b/java/forstjni/wal_filter_jnicallback.cc similarity index 97% rename from java/rocksjni/wal_filter_jnicallback.cc rename to java/forstjni/wal_filter_jnicallback.cc index d2e3c9076..aa5c2f31b 100644 --- a/java/rocksjni/wal_filter_jnicallback.cc +++ b/java/forstjni/wal_filter_jnicallback.cc @@ -6,10 +6,10 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::WalFilter. -#include "rocksjni/wal_filter_jnicallback.h" +#include "forstjni/wal_filter_jnicallback.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { WalFilterJniCallback::WalFilterJniCallback(JNIEnv* env, jobject jwal_filter) diff --git a/java/rocksjni/wal_filter_jnicallback.h b/java/forstjni/wal_filter_jnicallback.h similarity index 97% rename from java/rocksjni/wal_filter_jnicallback.h rename to java/forstjni/wal_filter_jnicallback.h index 5cdc65978..d933a2e8e 100644 --- a/java/rocksjni/wal_filter_jnicallback.h +++ b/java/forstjni/wal_filter_jnicallback.h @@ -16,7 +16,7 @@ #include #include "rocksdb/wal_filter.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" namespace ROCKSDB_NAMESPACE { diff --git a/java/rocksjni/write_batch.cc b/java/forstjni/write_batch.cc similarity index 83% rename from java/rocksjni/write_batch.cc rename to java/forstjni/write_batch.cc index 6704e4a7e..aa6c5f226 100644 --- a/java/rocksjni/write_batch.cc +++ b/java/forstjni/write_batch.cc @@ -11,25 +11,25 @@ #include "db/memtable.h" #include "db/write_batch_internal.h" -#include "include/org_rocksdb_WriteBatch.h" -#include "include/org_rocksdb_WriteBatch_Handler.h" +#include "include/org_forstdb_WriteBatch.h" +#include "include/org_forstdb_WriteBatch_Handler.h" #include "logging/logging.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/memtablerep.h" #include "rocksdb/status.h" #include "rocksdb/write_buffer_manager.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" -#include "rocksjni/writebatchhandlerjnicallback.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" +#include "forstjni/writebatchhandlerjnicallback.h" #include "table/scoped_arena_iterator.h" /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: newWriteBatch * Signature: (I)J */ -jlong Java_org_rocksdb_WriteBatch_newWriteBatch__I(JNIEnv* /*env*/, +jlong Java_org_forstdb_WriteBatch_newWriteBatch__I(JNIEnv* /*env*/, jclass /*jcls*/, jint jreserved_bytes) { auto* wb = @@ -38,11 +38,11 @@ jlong Java_org_rocksdb_WriteBatch_newWriteBatch__I(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: newWriteBatch * Signature: ([BI)J */ -jlong Java_org_rocksdb_WriteBatch_newWriteBatch___3BI(JNIEnv* env, +jlong Java_org_forstdb_WriteBatch_newWriteBatch___3BI(JNIEnv* env, jclass /*jcls*/, jbyteArray jserialized, jint jserialized_length) { @@ -61,11 +61,11 @@ jlong Java_org_rocksdb_WriteBatch_newWriteBatch___3BI(JNIEnv* env, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: count0 * Signature: (J)I */ -jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* /*env*/, jobject /*jobj*/, +jint Java_org_forstdb_WriteBatch_count0(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -74,11 +74,11 @@ jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: clear0 * Signature: (J)V */ -void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_WriteBatch_clear0(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -87,11 +87,11 @@ void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: setSavePoint0 * Signature: (J)V */ -void Java_org_rocksdb_WriteBatch_setSavePoint0(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBatch_setSavePoint0(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -101,11 +101,11 @@ void Java_org_rocksdb_WriteBatch_setSavePoint0(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: rollbackToSavePoint0 * Signature: (J)V */ -void Java_org_rocksdb_WriteBatch_rollbackToSavePoint0(JNIEnv* env, +void Java_org_forstdb_WriteBatch_rollbackToSavePoint0(JNIEnv* env, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -120,11 +120,11 @@ void Java_org_rocksdb_WriteBatch_rollbackToSavePoint0(JNIEnv* env, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: popSavePoint * Signature: (J)V */ -void Java_org_rocksdb_WriteBatch_popSavePoint(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_WriteBatch_popSavePoint(JNIEnv* env, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -138,11 +138,11 @@ void Java_org_rocksdb_WriteBatch_popSavePoint(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: setMaxBytes * Signature: (JJ)V */ -void Java_org_rocksdb_WriteBatch_setMaxBytes(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_WriteBatch_setMaxBytes(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle, jlong jmax_bytes) { auto* wb = reinterpret_cast(jwb_handle); @@ -152,11 +152,11 @@ void Java_org_rocksdb_WriteBatch_setMaxBytes(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: put * Signature: (J[BI[BI)V */ -void Java_org_rocksdb_WriteBatch_put__J_3BI_3BI(JNIEnv* env, jobject jobj, +void Java_org_forstdb_WriteBatch_put__J_3BI_3BI(JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, jbyteArray jentry_value, @@ -176,11 +176,11 @@ void Java_org_rocksdb_WriteBatch_put__J_3BI_3BI(JNIEnv* env, jobject jobj, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: put * Signature: (J[BI[BIJ)V */ -void Java_org_rocksdb_WriteBatch_put__J_3BI_3BIJ( +void Java_org_forstdb_WriteBatch_put__J_3BI_3BIJ( JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -201,11 +201,11 @@ void Java_org_rocksdb_WriteBatch_put__J_3BI_3BIJ( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: putDirect * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V */ -void Java_org_rocksdb_WriteBatch_putDirect(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_WriteBatch_putDirect(JNIEnv* env, jobject /*jobj*/, jlong jwb_handle, jobject jkey, jint jkey_offset, jint jkey_len, jobject jval, jint jval_offset, @@ -227,11 +227,11 @@ void Java_org_rocksdb_WriteBatch_putDirect(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: merge * Signature: (J[BI[BI)V */ -void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BI( +void Java_org_forstdb_WriteBatch_merge__J_3BI_3BI( JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) { auto* wb = reinterpret_cast(jwb_handle); @@ -249,11 +249,11 @@ void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BI( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: merge * Signature: (J[BI[BIJ)V */ -void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BIJ( +void Java_org_forstdb_WriteBatch_merge__J_3BI_3BIJ( JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -274,11 +274,11 @@ void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BIJ( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: delete * Signature: (J[BI)V */ -void Java_org_rocksdb_WriteBatch_delete__J_3BI(JNIEnv* env, jobject jobj, +void Java_org_forstdb_WriteBatch_delete__J_3BI(JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len) { auto* wb = reinterpret_cast(jwb_handle); @@ -292,11 +292,11 @@ void Java_org_rocksdb_WriteBatch_delete__J_3BI(JNIEnv* env, jobject jobj, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: delete * Signature: (J[BIJ)V */ -void Java_org_rocksdb_WriteBatch_delete__J_3BIJ(JNIEnv* env, jobject jobj, +void Java_org_forstdb_WriteBatch_delete__J_3BIJ(JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, jlong jcf_handle) { @@ -316,11 +316,11 @@ void Java_org_rocksdb_WriteBatch_delete__J_3BIJ(JNIEnv* env, jobject jobj, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: singleDelete * Signature: (J[BI)V */ -void Java_org_rocksdb_WriteBatch_singleDelete__J_3BI(JNIEnv* env, jobject jobj, +void Java_org_forstdb_WriteBatch_singleDelete__J_3BI(JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len) { @@ -338,11 +338,11 @@ void Java_org_rocksdb_WriteBatch_singleDelete__J_3BI(JNIEnv* env, jobject jobj, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: singleDelete * Signature: (J[BIJ)V */ -void Java_org_rocksdb_WriteBatch_singleDelete__J_3BIJ(JNIEnv* env, jobject jobj, +void Java_org_forstdb_WriteBatch_singleDelete__J_3BIJ(JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, @@ -364,11 +364,11 @@ void Java_org_rocksdb_WriteBatch_singleDelete__J_3BIJ(JNIEnv* env, jobject jobj, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: deleteDirect * Signature: (JLjava/nio/ByteBuffer;IIJ)V */ -void Java_org_rocksdb_WriteBatch_deleteDirect(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_WriteBatch_deleteDirect(JNIEnv* env, jobject /*jobj*/, jlong jwb_handle, jobject jkey, jint jkey_offset, jint jkey_len, jlong jcf_handle) { @@ -388,11 +388,11 @@ void Java_org_rocksdb_WriteBatch_deleteDirect(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: deleteRange * Signature: (J[BI[BI)V */ -void Java_org_rocksdb_WriteBatch_deleteRange__J_3BI_3BI( +void Java_org_forstdb_WriteBatch_deleteRange__J_3BI_3BI( JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jbegin_key, jint jbegin_key_len, jbyteArray jend_key, jint jend_key_len) { auto* wb = reinterpret_cast(jwb_handle); @@ -410,11 +410,11 @@ void Java_org_rocksdb_WriteBatch_deleteRange__J_3BI_3BI( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: deleteRange * Signature: (J[BI[BIJ)V */ -void Java_org_rocksdb_WriteBatch_deleteRange__J_3BI_3BIJ( +void Java_org_forstdb_WriteBatch_deleteRange__J_3BI_3BIJ( JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jbegin_key, jint jbegin_key_len, jbyteArray jend_key, jint jend_key_len, jlong jcf_handle) { @@ -436,11 +436,11 @@ void Java_org_rocksdb_WriteBatch_deleteRange__J_3BI_3BIJ( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: putLogData * Signature: (J[BI)V */ -void Java_org_rocksdb_WriteBatch_putLogData(JNIEnv* env, jobject jobj, +void Java_org_forstdb_WriteBatch_putLogData(JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jblob, jint jblob_len) { auto* wb = reinterpret_cast(jwb_handle); @@ -456,11 +456,11 @@ void Java_org_rocksdb_WriteBatch_putLogData(JNIEnv* env, jobject jobj, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: iterate * Signature: (JJ)V */ -void Java_org_rocksdb_WriteBatch_iterate(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_WriteBatch_iterate(JNIEnv* env, jobject /*jobj*/, jlong jwb_handle, jlong handlerHandle) { auto* wb = reinterpret_cast(jwb_handle); @@ -477,11 +477,11 @@ void Java_org_rocksdb_WriteBatch_iterate(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: data * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_WriteBatch_data(JNIEnv* env, jobject /*jobj*/, +jbyteArray Java_org_forstdb_WriteBatch_data(JNIEnv* env, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -491,11 +491,11 @@ jbyteArray Java_org_rocksdb_WriteBatch_data(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: getDataSize * Signature: (J)J */ -jlong Java_org_rocksdb_WriteBatch_getDataSize(JNIEnv* /*env*/, jobject /*jobj*/, +jlong Java_org_forstdb_WriteBatch_getDataSize(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -505,11 +505,11 @@ jlong Java_org_rocksdb_WriteBatch_getDataSize(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasPut * Signature: (J)Z */ -jboolean Java_org_rocksdb_WriteBatch_hasPut(JNIEnv* /*env*/, jobject /*jobj*/, +jboolean Java_org_forstdb_WriteBatch_hasPut(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -518,11 +518,11 @@ jboolean Java_org_rocksdb_WriteBatch_hasPut(JNIEnv* /*env*/, jobject /*jobj*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasDelete * Signature: (J)Z */ -jboolean Java_org_rocksdb_WriteBatch_hasDelete(JNIEnv* /*env*/, +jboolean Java_org_forstdb_WriteBatch_hasDelete(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -532,11 +532,11 @@ jboolean Java_org_rocksdb_WriteBatch_hasDelete(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasSingleDelete * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasSingleDelete( +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasSingleDelete( JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -545,11 +545,11 @@ JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasSingleDelete( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasDeleteRange * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasDeleteRange( +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasDeleteRange( JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -558,11 +558,11 @@ JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasDeleteRange( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasMerge * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasMerge( +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasMerge( JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -571,11 +571,11 @@ JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasMerge( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasBeginPrepare * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasBeginPrepare( +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasBeginPrepare( JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -584,11 +584,11 @@ JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasBeginPrepare( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasEndPrepare * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasEndPrepare( +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasEndPrepare( JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -597,11 +597,11 @@ JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasEndPrepare( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasCommit * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasCommit( +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasCommit( JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -610,11 +610,11 @@ JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasCommit( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: hasRollback * Signature: (J)Z */ -JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasRollback( +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasRollback( JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -623,11 +623,11 @@ JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasRollback( } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: markWalTerminationPoint * Signature: (J)V */ -void Java_org_rocksdb_WriteBatch_markWalTerminationPoint(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBatch_markWalTerminationPoint(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -637,11 +637,11 @@ void Java_org_rocksdb_WriteBatch_markWalTerminationPoint(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: getWalTerminationPoint - * Signature: (J)Lorg/rocksdb/WriteBatch/SavePoint; + * Signature: (J)Lorg/forstdb/WriteBatch/SavePoint; */ -jobject Java_org_rocksdb_WriteBatch_getWalTerminationPoint(JNIEnv* env, +jobject Java_org_forstdb_WriteBatch_getWalTerminationPoint(JNIEnv* env, jobject /*jobj*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -652,11 +652,11 @@ jobject Java_org_rocksdb_WriteBatch_getWalTerminationPoint(JNIEnv* env, } /* - * Class: org_rocksdb_WriteBatch + * Class: org_forstdb_WriteBatch * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_WriteBatch_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBatch_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { auto* wb = reinterpret_cast(handle); @@ -665,11 +665,11 @@ void Java_org_rocksdb_WriteBatch_disposeInternal(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatch_Handler + * Class: org_forstdb_WriteBatch_Handler * Method: createNewHandler0 * Signature: ()J */ -jlong Java_org_rocksdb_WriteBatch_00024Handler_createNewHandler0(JNIEnv* env, +jlong Java_org_forstdb_WriteBatch_00024Handler_createNewHandler0(JNIEnv* env, jobject jobj) { auto* wbjnic = new ROCKSDB_NAMESPACE::WriteBatchHandlerJniCallback(env, jobj); return GET_CPLUSPLUS_POINTER(wbjnic); diff --git a/java/rocksjni/write_batch_test.cc b/java/forstjni/write_batch_test.cc similarity index 90% rename from java/rocksjni/write_batch_test.cc rename to java/forstjni/write_batch_test.cc index 30b9a7229..bf3669d0c 100644 --- a/java/rocksjni/write_batch_test.cc +++ b/java/forstjni/write_batch_test.cc @@ -11,27 +11,27 @@ #include "db/memtable.h" #include "db/write_batch_internal.h" -#include "include/org_rocksdb_WriteBatch.h" -#include "include/org_rocksdb_WriteBatchTest.h" -#include "include/org_rocksdb_WriteBatchTestInternalHelper.h" -#include "include/org_rocksdb_WriteBatch_Handler.h" +#include "include/org_forstdb_WriteBatch.h" +#include "include/org_forstdb_WriteBatchTest.h" +#include "include/org_forstdb_WriteBatchTestInternalHelper.h" +#include "include/org_forstdb_WriteBatch_Handler.h" #include "options/cf_options.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/memtablerep.h" #include "rocksdb/status.h" #include "rocksdb/write_buffer_manager.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" #include "table/scoped_arena_iterator.h" #include "test_util/testharness.h" #include "util/string_util.h" /* - * Class: org_rocksdb_WriteBatchTest + * Class: org_forstdb_WriteBatchTest * Method: getContents * Signature: (J)[B */ -jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(JNIEnv* env, +jbyteArray Java_org_forstdb_WriteBatchTest_getContents(JNIEnv* env, jclass /*jclazz*/, jlong jwb_handle) { auto* b = reinterpret_cast(jwb_handle); @@ -153,11 +153,11 @@ jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(JNIEnv* env, } /* - * Class: org_rocksdb_WriteBatchTestInternalHelper + * Class: org_forstdb_WriteBatchTestInternalHelper * Method: setSequence * Signature: (JJ)V */ -void Java_org_rocksdb_WriteBatchTestInternalHelper_setSequence( +void Java_org_forstdb_WriteBatchTestInternalHelper_setSequence( JNIEnv* /*env*/, jclass /*jclazz*/, jlong jwb_handle, jlong jsn) { auto* wb = reinterpret_cast(jwb_handle); assert(wb != nullptr); @@ -167,11 +167,11 @@ void Java_org_rocksdb_WriteBatchTestInternalHelper_setSequence( } /* - * Class: org_rocksdb_WriteBatchTestInternalHelper + * Class: org_forstdb_WriteBatchTestInternalHelper * Method: sequence * Signature: (J)J */ -jlong Java_org_rocksdb_WriteBatchTestInternalHelper_sequence(JNIEnv* /*env*/, +jlong Java_org_forstdb_WriteBatchTestInternalHelper_sequence(JNIEnv* /*env*/, jclass /*jclazz*/, jlong jwb_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -182,11 +182,11 @@ jlong Java_org_rocksdb_WriteBatchTestInternalHelper_sequence(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatchTestInternalHelper + * Class: org_forstdb_WriteBatchTestInternalHelper * Method: append * Signature: (JJ)V */ -void Java_org_rocksdb_WriteBatchTestInternalHelper_append(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBatchTestInternalHelper_append(JNIEnv* /*env*/, jclass /*jclazz*/, jlong jwb_handle_1, jlong jwb_handle_2) { diff --git a/java/rocksjni/write_batch_with_index.cc b/java/forstjni/write_batch_with_index.cc similarity index 84% rename from java/rocksjni/write_batch_with_index.cc rename to java/forstjni/write_batch_with_index.cc index a5c3216cb..e13c750e1 100644 --- a/java/rocksjni/write_batch_with_index.cc +++ b/java/forstjni/write_batch_with_index.cc @@ -8,29 +8,29 @@ #include "rocksdb/utilities/write_batch_with_index.h" -#include "include/org_rocksdb_WBWIRocksIterator.h" -#include "include/org_rocksdb_WriteBatchWithIndex.h" +#include "include/org_forstdb_WBWIRocksIterator.h" +#include "include/org_forstdb_WriteBatchWithIndex.h" #include "rocksdb/comparator.h" -#include "rocksjni/cplusplus_to_java_convert.h" -#include "rocksjni/portal.h" +#include "forstjni/cplusplus_to_java_convert.h" +#include "forstjni/portal.h" /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: newWriteBatchWithIndex * Signature: ()J */ -jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__( +jlong Java_org_forstdb_WriteBatchWithIndex_newWriteBatchWithIndex__( JNIEnv* /*env*/, jclass /*jcls*/) { auto* wbwi = new ROCKSDB_NAMESPACE::WriteBatchWithIndex(); return GET_CPLUSPLUS_POINTER(wbwi); } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: newWriteBatchWithIndex * Signature: (Z)J */ -jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z( +jlong Java_org_forstdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z( JNIEnv* /*env*/, jclass /*jcls*/, jboolean joverwrite_key) { auto* wbwi = new ROCKSDB_NAMESPACE::WriteBatchWithIndex( ROCKSDB_NAMESPACE::BytewiseComparator(), 0, @@ -39,11 +39,11 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: newWriteBatchWithIndex * Signature: (JBIZ)J */ -jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__JBIZ( +jlong Java_org_forstdb_WriteBatchWithIndex_newWriteBatchWithIndex__JBIZ( JNIEnv* /*env*/, jclass /*jcls*/, jlong jfallback_index_comparator_handle, jbyte jcomparator_type, jint jreserved_bytes, jboolean joverwrite_key) { ROCKSDB_NAMESPACE::Comparator* fallback_comparator = nullptr; @@ -68,11 +68,11 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__JBIZ( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: count0 * Signature: (J)I */ -jint Java_org_rocksdb_WriteBatchWithIndex_count0(JNIEnv* /*env*/, +jint Java_org_forstdb_WriteBatchWithIndex_count0(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwbwi_handle) { auto* wbwi = @@ -83,11 +83,11 @@ jint Java_org_rocksdb_WriteBatchWithIndex_count0(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: put * Signature: (J[BI[BI)V */ -void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BI( +void Java_org_forstdb_WriteBatchWithIndex_put__J_3BI_3BI( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) { auto* wbwi = @@ -106,11 +106,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BI( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: put * Signature: (J[BI[BIJ)V */ -void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BIJ( +void Java_org_forstdb_WriteBatchWithIndex_put__J_3BI_3BIJ( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, jint jkey_len, jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) { @@ -133,11 +133,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BIJ( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: putDirect * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V */ -void Java_org_rocksdb_WriteBatchWithIndex_putDirect( +void Java_org_forstdb_WriteBatchWithIndex_putDirect( JNIEnv* env, jobject /*jobj*/, jlong jwb_handle, jobject jkey, jint jkey_offset, jint jkey_len, jobject jval, jint jval_offset, jint jval_len, jlong jcf_handle) { @@ -158,11 +158,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_putDirect( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: merge * Signature: (J[BI[BI)V */ -void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BI( +void Java_org_forstdb_WriteBatchWithIndex_merge__J_3BI_3BI( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) { auto* wbwi = @@ -181,11 +181,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BI( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: merge * Signature: (J[BI[BIJ)V */ -void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BIJ( +void Java_org_forstdb_WriteBatchWithIndex_merge__J_3BI_3BIJ( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, jint jkey_len, jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) { @@ -208,11 +208,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BIJ( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: delete * Signature: (J[BI)V */ -void Java_org_rocksdb_WriteBatchWithIndex_delete__J_3BI(JNIEnv* env, +void Java_org_forstdb_WriteBatchWithIndex_delete__J_3BI(JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, @@ -231,11 +231,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_delete__J_3BI(JNIEnv* env, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: delete * Signature: (J[BIJ)V */ -void Java_org_rocksdb_WriteBatchWithIndex_delete__J_3BIJ( +void Java_org_forstdb_WriteBatchWithIndex_delete__J_3BIJ( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, jint jkey_len, jlong jcf_handle) { auto* wbwi = @@ -255,11 +255,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_delete__J_3BIJ( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: singleDelete * Signature: (J[BI)V */ -void Java_org_rocksdb_WriteBatchWithIndex_singleDelete__J_3BI( +void Java_org_forstdb_WriteBatchWithIndex_singleDelete__J_3BI( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, jint jkey_len) { auto* wbwi = @@ -277,11 +277,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_singleDelete__J_3BI( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: singleDelete * Signature: (J[BIJ)V */ -void Java_org_rocksdb_WriteBatchWithIndex_singleDelete__J_3BIJ( +void Java_org_forstdb_WriteBatchWithIndex_singleDelete__J_3BIJ( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, jint jkey_len, jlong jcf_handle) { auto* wbwi = @@ -302,11 +302,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_singleDelete__J_3BIJ( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: deleteDirect * Signature: (JLjava/nio/ByteBuffer;IIJ)V */ -void Java_org_rocksdb_WriteBatchWithIndex_deleteDirect( +void Java_org_forstdb_WriteBatchWithIndex_deleteDirect( JNIEnv* env, jobject /*jobj*/, jlong jwb_handle, jobject jkey, jint jkey_offset, jint jkey_len, jlong jcf_handle) { auto* wb = reinterpret_cast(jwb_handle); @@ -325,11 +325,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_deleteDirect( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: deleteRange * Signature: (J[BI[BI)V */ -void Java_org_rocksdb_WriteBatchWithIndex_deleteRange__J_3BI_3BI( +void Java_org_forstdb_WriteBatchWithIndex_deleteRange__J_3BI_3BI( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jbegin_key, jint jbegin_key_len, jbyteArray jend_key, jint jend_key_len) { auto* wbwi = @@ -348,11 +348,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_deleteRange__J_3BI_3BI( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: deleteRange * Signature: (J[BI[BIJ)V */ -void Java_org_rocksdb_WriteBatchWithIndex_deleteRange__J_3BI_3BIJ( +void Java_org_forstdb_WriteBatchWithIndex_deleteRange__J_3BI_3BIJ( JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jbegin_key, jint jbegin_key_len, jbyteArray jend_key, jint jend_key_len, jlong jcf_handle) { @@ -375,11 +375,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_deleteRange__J_3BI_3BIJ( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: putLogData * Signature: (J[BI)V */ -void Java_org_rocksdb_WriteBatchWithIndex_putLogData(JNIEnv* env, jobject jobj, +void Java_org_forstdb_WriteBatchWithIndex_putLogData(JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jblob, jint jblob_len) { @@ -397,11 +397,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_putLogData(JNIEnv* env, jobject jobj, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: clear * Signature: (J)V */ -void Java_org_rocksdb_WriteBatchWithIndex_clear0(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBatchWithIndex_clear0(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwbwi_handle) { auto* wbwi = @@ -412,11 +412,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_clear0(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: setSavePoint0 * Signature: (J)V */ -void Java_org_rocksdb_WriteBatchWithIndex_setSavePoint0(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBatchWithIndex_setSavePoint0(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwbwi_handle) { auto* wbwi = @@ -427,11 +427,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_setSavePoint0(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: rollbackToSavePoint0 * Signature: (J)V */ -void Java_org_rocksdb_WriteBatchWithIndex_rollbackToSavePoint0( +void Java_org_forstdb_WriteBatchWithIndex_rollbackToSavePoint0( JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle) { auto* wbwi = reinterpret_cast(jwbwi_handle); @@ -447,11 +447,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_rollbackToSavePoint0( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: popSavePoint * Signature: (J)V */ -void Java_org_rocksdb_WriteBatchWithIndex_popSavePoint(JNIEnv* env, +void Java_org_forstdb_WriteBatchWithIndex_popSavePoint(JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle) { auto* wbwi = @@ -468,11 +468,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_popSavePoint(JNIEnv* env, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: setMaxBytes * Signature: (JJ)V */ -void Java_org_rocksdb_WriteBatchWithIndex_setMaxBytes(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBatchWithIndex_setMaxBytes(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwbwi_handle, jlong jmax_bytes) { @@ -484,11 +484,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_setMaxBytes(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: getWriteBatch - * Signature: (J)Lorg/rocksdb/WriteBatch; + * Signature: (J)Lorg/forstdb/WriteBatch; */ -jobject Java_org_rocksdb_WriteBatchWithIndex_getWriteBatch(JNIEnv* env, +jobject Java_org_forstdb_WriteBatchWithIndex_getWriteBatch(JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle) { auto* wbwi = @@ -502,11 +502,11 @@ jobject Java_org_rocksdb_WriteBatchWithIndex_getWriteBatch(JNIEnv* env, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: iterator0 * Signature: (J)J */ -jlong Java_org_rocksdb_WriteBatchWithIndex_iterator0(JNIEnv* /*env*/, +jlong Java_org_forstdb_WriteBatchWithIndex_iterator0(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwbwi_handle) { auto* wbwi = @@ -516,11 +516,11 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_iterator0(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: iterator1 * Signature: (JJ)J */ -jlong Java_org_rocksdb_WriteBatchWithIndex_iterator1(JNIEnv* /*env*/, +jlong Java_org_forstdb_WriteBatchWithIndex_iterator1(JNIEnv* /*env*/, jobject /*jobj*/, jlong jwbwi_handle, jlong jcf_handle) { @@ -533,11 +533,11 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_iterator1(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: iteratorWithBase * Signature: (JJJJ)J */ -jlong Java_org_rocksdb_WriteBatchWithIndex_iteratorWithBase( +jlong Java_org_forstdb_WriteBatchWithIndex_iteratorWithBase( JNIEnv*, jobject, jlong jwbwi_handle, jlong jcf_handle, jlong jbase_iterator_handle, jlong jread_opts_handle) { auto* wbwi = @@ -557,11 +557,11 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_iteratorWithBase( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: getFromBatch * Signature: (JJ[BI)[B */ -jbyteArray JNICALL Java_org_rocksdb_WriteBatchWithIndex_getFromBatch__JJ_3BI( +jbyteArray JNICALL Java_org_forstdb_WriteBatchWithIndex_getFromBatch__JJ_3BI( JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle, jlong jdbopt_handle, jbyteArray jkey, jint jkey_len) { auto* wbwi = @@ -577,11 +577,11 @@ jbyteArray JNICALL Java_org_rocksdb_WriteBatchWithIndex_getFromBatch__JJ_3BI( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: getFromBatch * Signature: (JJ[BIJ)[B */ -jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatch__JJ_3BIJ( +jbyteArray Java_org_forstdb_WriteBatchWithIndex_getFromBatch__JJ_3BIJ( JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle, jlong jdbopt_handle, jbyteArray jkey, jint jkey_len, jlong jcf_handle) { auto* wbwi = @@ -599,11 +599,11 @@ jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatch__JJ_3BIJ( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: getFromBatchAndDB * Signature: (JJJ[BI)[B */ -jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BI( +jbyteArray Java_org_forstdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BI( JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle, jlong jdb_handle, jlong jreadopt_handle, jbyteArray jkey, jint jkey_len) { auto* wbwi = @@ -621,11 +621,11 @@ jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BI( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: getFromBatchAndDB * Signature: (JJJ[BIJ)[B */ -jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BIJ( +jbyteArray Java_org_forstdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BIJ( JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle, jlong jdb_handle, jlong jreadopt_handle, jbyteArray jkey, jint jkey_len, jlong jcf_handle) { auto* wbwi = @@ -645,11 +645,11 @@ jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BIJ( } /* - * Class: org_rocksdb_WriteBatchWithIndex + * Class: org_forstdb_WriteBatchWithIndex * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_WriteBatchWithIndex_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBatchWithIndex_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { auto* wbwi = @@ -661,11 +661,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_disposeInternal(JNIEnv* /*env*/, /* WBWIRocksIterator below */ /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_WBWIRocksIterator_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_WBWIRocksIterator_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); @@ -674,64 +674,64 @@ void Java_org_rocksdb_WBWIRocksIterator_disposeInternal(JNIEnv* /*env*/, } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: isValid0 * Signature: (J)Z */ -jboolean Java_org_rocksdb_WBWIRocksIterator_isValid0(JNIEnv* /*env*/, +jboolean Java_org_forstdb_WBWIRocksIterator_isValid0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { return reinterpret_cast(handle)->Valid(); } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: seekToFirst0 * Signature: (J)V */ -void Java_org_rocksdb_WBWIRocksIterator_seekToFirst0(JNIEnv* /*env*/, +void Java_org_forstdb_WBWIRocksIterator_seekToFirst0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->SeekToFirst(); } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: seekToLast0 * Signature: (J)V */ -void Java_org_rocksdb_WBWIRocksIterator_seekToLast0(JNIEnv* /*env*/, +void Java_org_forstdb_WBWIRocksIterator_seekToLast0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->SeekToLast(); } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: next0 * Signature: (J)V */ -void Java_org_rocksdb_WBWIRocksIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_WBWIRocksIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->Next(); } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: prev0 * Signature: (J)V */ -void Java_org_rocksdb_WBWIRocksIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, +void Java_org_forstdb_WBWIRocksIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, jlong handle) { reinterpret_cast(handle)->Prev(); } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: seek0 * Signature: (J[BI)V */ -void Java_org_rocksdb_WBWIRocksIterator_seek0(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_WBWIRocksIterator_seek0(JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -752,11 +752,11 @@ void Java_org_rocksdb_WBWIRocksIterator_seek0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: seekDirect0 * Signature: (JLjava/nio/ByteBuffer;II)V */ -void Java_org_rocksdb_WBWIRocksIterator_seekDirect0( +void Java_org_forstdb_WBWIRocksIterator_seekDirect0( JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -771,11 +771,11 @@ void Java_org_rocksdb_WBWIRocksIterator_seekDirect0( * This method supports fetching into indirect byte buffers; * the Java wrapper extracts the byte[] and passes it here. * - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: seekByteArray0 * Signature: (J[BII)V */ -void Java_org_rocksdb_WBWIRocksIterator_seekByteArray0( +void Java_org_forstdb_WBWIRocksIterator_seekByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_off, jint jtarget_len) { const std::unique_ptr target(new char[jtarget_len]); @@ -795,11 +795,11 @@ void Java_org_rocksdb_WBWIRocksIterator_seekByteArray0( } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: seekForPrev0 * Signature: (J[BI)V */ -void Java_org_rocksdb_WBWIRocksIterator_seekForPrev0(JNIEnv* env, +void Java_org_forstdb_WBWIRocksIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, @@ -822,11 +822,11 @@ void Java_org_rocksdb_WBWIRocksIterator_seekForPrev0(JNIEnv* env, } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: seekForPrevDirect0 * Signature: (JLjava/nio/ByteBuffer;II)V */ -void Java_org_rocksdb_WBWIRocksIterator_seekForPrevDirect0( +void Java_org_forstdb_WBWIRocksIterator_seekForPrevDirect0( JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, jint jtarget_off, jint jtarget_len) { auto* it = reinterpret_cast(handle); @@ -841,11 +841,11 @@ void Java_org_rocksdb_WBWIRocksIterator_seekForPrevDirect0( * This method supports fetching into indirect byte buffers; * the Java wrapper extracts the byte[] and passes it here. * - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: seekForPrevByteArray0 * Signature: (J[BII)V */ -void Java_org_rocksdb_WBWIRocksIterator_seekForPrevByteArray0( +void Java_org_forstdb_WBWIRocksIterator_seekForPrevByteArray0( JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, jint jtarget_off, jint jtarget_len) { const std::unique_ptr target(new char[jtarget_len]); @@ -865,11 +865,11 @@ void Java_org_rocksdb_WBWIRocksIterator_seekForPrevByteArray0( } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: status0 * Signature: (J)V */ -void Java_org_rocksdb_WBWIRocksIterator_status0(JNIEnv* env, jobject /*jobj*/, +void Java_org_forstdb_WBWIRocksIterator_status0(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); ROCKSDB_NAMESPACE::Status s = it->status(); @@ -882,11 +882,11 @@ void Java_org_rocksdb_WBWIRocksIterator_status0(JNIEnv* env, jobject /*jobj*/, } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: entry1 * Signature: (J)[J */ -jlongArray Java_org_rocksdb_WBWIRocksIterator_entry1(JNIEnv* env, +jlongArray Java_org_forstdb_WBWIRocksIterator_entry1(JNIEnv* env, jobject /*jobj*/, jlong handle) { auto* it = reinterpret_cast(handle); @@ -898,7 +898,7 @@ jlongArray Java_org_rocksdb_WBWIRocksIterator_entry1(JNIEnv* env, results[0] = ROCKSDB_NAMESPACE::WriteTypeJni::toJavaWriteType(we.type); // NOTE: key_slice and value_slice will be freed by - // org.rocksdb.DirectSlice#close + // org.forstdb.DirectSlice#close auto* key_slice = new ROCKSDB_NAMESPACE::Slice(we.key.data(), we.key.size()); results[1] = GET_CPLUSPLUS_POINTER(key_slice); @@ -942,11 +942,11 @@ jlongArray Java_org_rocksdb_WBWIRocksIterator_entry1(JNIEnv* env, } /* - * Class: org_rocksdb_WBWIRocksIterator + * Class: org_forstdb_WBWIRocksIterator * Method: refresh0 * Signature: (J)V */ -void Java_org_rocksdb_WBWIRocksIterator_refresh0(JNIEnv* env) { +void Java_org_forstdb_WBWIRocksIterator_refresh0(JNIEnv* env) { ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::Status::NotSupported("Refresh() is not supported"); ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); diff --git a/java/rocksjni/write_buffer_manager.cc b/java/forstjni/write_buffer_manager.cc similarity index 81% rename from java/rocksjni/write_buffer_manager.cc rename to java/forstjni/write_buffer_manager.cc index 9ce697e10..114d3a64b 100644 --- a/java/rocksjni/write_buffer_manager.cc +++ b/java/forstjni/write_buffer_manager.cc @@ -9,16 +9,16 @@ #include -#include "include/org_rocksdb_WriteBufferManager.h" +#include "include/org_forstdb_WriteBufferManager.h" #include "rocksdb/cache.h" -#include "rocksjni/cplusplus_to_java_convert.h" +#include "forstjni/cplusplus_to_java_convert.h" /* - * Class: org_rocksdb_WriteBufferManager + * Class: org_forstdb_WriteBufferManager * Method: newWriteBufferManager * Signature: (JJ)J */ -jlong Java_org_rocksdb_WriteBufferManager_newWriteBufferManager( +jlong Java_org_forstdb_WriteBufferManager_newWriteBufferManager( JNIEnv* /*env*/, jclass /*jclazz*/, jlong jbuffer_size, jlong jcache_handle, jboolean allow_stall) { auto* cache_ptr = @@ -32,11 +32,11 @@ jlong Java_org_rocksdb_WriteBufferManager_newWriteBufferManager( } /* - * Class: org_rocksdb_WriteBufferManager + * Class: org_forstdb_WriteBufferManager * Method: disposeInternal * Signature: (J)V */ -void Java_org_rocksdb_WriteBufferManager_disposeInternal(JNIEnv* /*env*/, +void Java_org_forstdb_WriteBufferManager_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { auto* write_buffer_manager = diff --git a/java/rocksjni/writebatchhandlerjnicallback.cc b/java/forstjni/writebatchhandlerjnicallback.cc similarity index 99% rename from java/rocksjni/writebatchhandlerjnicallback.cc rename to java/forstjni/writebatchhandlerjnicallback.cc index 66ceabe9a..04e97f8bd 100644 --- a/java/rocksjni/writebatchhandlerjnicallback.cc +++ b/java/forstjni/writebatchhandlerjnicallback.cc @@ -6,9 +6,9 @@ // This file implements the callback "bridge" between Java and C++ for // ROCKSDB_NAMESPACE::Comparator. -#include "rocksjni/writebatchhandlerjnicallback.h" +#include "forstjni/writebatchhandlerjnicallback.h" -#include "rocksjni/portal.h" +#include "forstjni/portal.h" namespace ROCKSDB_NAMESPACE { WriteBatchHandlerJniCallback::WriteBatchHandlerJniCallback( diff --git a/java/rocksjni/writebatchhandlerjnicallback.h b/java/forstjni/writebatchhandlerjnicallback.h similarity index 99% rename from java/rocksjni/writebatchhandlerjnicallback.h rename to java/forstjni/writebatchhandlerjnicallback.h index 9629797ca..b71935ad3 100644 --- a/java/rocksjni/writebatchhandlerjnicallback.h +++ b/java/forstjni/writebatchhandlerjnicallback.h @@ -15,7 +15,7 @@ #include #include "rocksdb/write_batch.h" -#include "rocksjni/jnicallback.h" +#include "forstjni/jnicallback.h" namespace ROCKSDB_NAMESPACE { /** diff --git a/java/include/org_forstdb_AbstractCompactionFilter.h b/java/include/org_forstdb_AbstractCompactionFilter.h new file mode 100644 index 000000000..65ae517f7 --- /dev/null +++ b/java/include/org_forstdb_AbstractCompactionFilter.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractCompactionFilter */ + +#ifndef _Included_org_forstdb_AbstractCompactionFilter +#define _Included_org_forstdb_AbstractCompactionFilter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractCompactionFilter + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_AbstractCompactionFilter_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_AbstractCompactionFilterFactory.h b/java/include/org_forstdb_AbstractCompactionFilterFactory.h new file mode 100644 index 000000000..1884a297d --- /dev/null +++ b/java/include/org_forstdb_AbstractCompactionFilterFactory.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractCompactionFilterFactory */ + +#ifndef _Included_org_forstdb_AbstractCompactionFilterFactory +#define _Included_org_forstdb_AbstractCompactionFilterFactory +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractCompactionFilterFactory + * Method: createNewCompactionFilterFactory0 + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_AbstractCompactionFilterFactory_createNewCompactionFilterFactory0 + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_AbstractCompactionFilterFactory + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_AbstractCompactionFilterFactory_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_AbstractComparator.h b/java/include/org_forstdb_AbstractComparator.h new file mode 100644 index 000000000..d476fdbe7 --- /dev/null +++ b/java/include/org_forstdb_AbstractComparator.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractComparator */ + +#ifndef _Included_org_forstdb_AbstractComparator +#define _Included_org_forstdb_AbstractComparator +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractComparator + * Method: usingDirectBuffers + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_AbstractComparator_usingDirectBuffers + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_AbstractComparator + * Method: createNewComparator + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_AbstractComparator_createNewComparator + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_AbstractEventListener.h b/java/include/org_forstdb_AbstractEventListener.h new file mode 100644 index 000000000..e04648a8e --- /dev/null +++ b/java/include/org_forstdb_AbstractEventListener.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractEventListener */ + +#ifndef _Included_org_forstdb_AbstractEventListener +#define _Included_org_forstdb_AbstractEventListener +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractEventListener + * Method: createNewEventListener + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_AbstractEventListener_createNewEventListener + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_AbstractEventListener + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_AbstractEventListener_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_AbstractSlice.h b/java/include/org_forstdb_AbstractSlice.h new file mode 100644 index 000000000..2121b1fe3 --- /dev/null +++ b/java/include/org_forstdb_AbstractSlice.h @@ -0,0 +1,69 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractSlice */ + +#ifndef _Included_org_forstdb_AbstractSlice +#define _Included_org_forstdb_AbstractSlice +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractSlice + * Method: createNewSliceFromString + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_AbstractSlice_createNewSliceFromString + (JNIEnv *, jclass, jstring); + +/* + * Class: org_forstdb_AbstractSlice + * Method: size0 + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_AbstractSlice_size0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_AbstractSlice + * Method: empty0 + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_AbstractSlice_empty0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_AbstractSlice + * Method: toString0 + * Signature: (JZ)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_AbstractSlice_toString0 + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_AbstractSlice + * Method: compare0 + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_AbstractSlice_compare0 + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_AbstractSlice + * Method: startsWith0 + * Signature: (JJ)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_AbstractSlice_startsWith0 + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_AbstractSlice + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_AbstractSlice_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_AbstractTableFilter.h b/java/include/org_forstdb_AbstractTableFilter.h new file mode 100644 index 000000000..35fa3f360 --- /dev/null +++ b/java/include/org_forstdb_AbstractTableFilter.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractTableFilter */ + +#ifndef _Included_org_forstdb_AbstractTableFilter +#define _Included_org_forstdb_AbstractTableFilter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractTableFilter + * Method: createNewTableFilter + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_AbstractTableFilter_createNewTableFilter + (JNIEnv *, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_AbstractTraceWriter.h b/java/include/org_forstdb_AbstractTraceWriter.h new file mode 100644 index 000000000..820d6fe0d --- /dev/null +++ b/java/include/org_forstdb_AbstractTraceWriter.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractTraceWriter */ + +#ifndef _Included_org_forstdb_AbstractTraceWriter +#define _Included_org_forstdb_AbstractTraceWriter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractTraceWriter + * Method: createNewTraceWriter + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_AbstractTraceWriter_createNewTraceWriter + (JNIEnv *, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_AbstractTransactionNotifier.h b/java/include/org_forstdb_AbstractTransactionNotifier.h new file mode 100644 index 000000000..b43bad529 --- /dev/null +++ b/java/include/org_forstdb_AbstractTransactionNotifier.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractTransactionNotifier */ + +#ifndef _Included_org_forstdb_AbstractTransactionNotifier +#define _Included_org_forstdb_AbstractTransactionNotifier +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractTransactionNotifier + * Method: createNewTransactionNotifier + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_AbstractTransactionNotifier_createNewTransactionNotifier + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_AbstractTransactionNotifier + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_AbstractTransactionNotifier_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_AbstractWalFilter.h b/java/include/org_forstdb_AbstractWalFilter.h new file mode 100644 index 000000000..ff7094403 --- /dev/null +++ b/java/include/org_forstdb_AbstractWalFilter.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_AbstractWalFilter */ + +#ifndef _Included_org_forstdb_AbstractWalFilter +#define _Included_org_forstdb_AbstractWalFilter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_AbstractWalFilter + * Method: createNewWalFilter + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_AbstractWalFilter_createNewWalFilter + (JNIEnv *, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_BackupEngine.h b/java/include/org_forstdb_BackupEngine.h new file mode 100644 index 000000000..a88572dd1 --- /dev/null +++ b/java/include/org_forstdb_BackupEngine.h @@ -0,0 +1,101 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_BackupEngine */ + +#ifndef _Included_org_forstdb_BackupEngine +#define _Included_org_forstdb_BackupEngine +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_BackupEngine + * Method: open + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_BackupEngine_open + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_BackupEngine + * Method: createNewBackup + * Signature: (JJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngine_createNewBackup + (JNIEnv *, jobject, jlong, jlong, jboolean); + +/* + * Class: org_forstdb_BackupEngine + * Method: createNewBackupWithMetadata + * Signature: (JJLjava/lang/String;Z)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngine_createNewBackupWithMetadata + (JNIEnv *, jobject, jlong, jlong, jstring, jboolean); + +/* + * Class: org_forstdb_BackupEngine + * Method: getBackupInfo + * Signature: (J)Ljava/util/List; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_BackupEngine_getBackupInfo + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngine + * Method: getCorruptedBackups + * Signature: (J)[I + */ +JNIEXPORT jintArray JNICALL Java_org_forstdb_BackupEngine_getCorruptedBackups + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngine + * Method: garbageCollect + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngine_garbageCollect + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngine + * Method: purgeOldBackups + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngine_purgeOldBackups + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_BackupEngine + * Method: deleteBackup + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngine_deleteBackup + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_BackupEngine + * Method: restoreDbFromBackup + * Signature: (JILjava/lang/String;Ljava/lang/String;J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngine_restoreDbFromBackup + (JNIEnv *, jobject, jlong, jint, jstring, jstring, jlong); + +/* + * Class: org_forstdb_BackupEngine + * Method: restoreDbFromLatestBackup + * Signature: (JLjava/lang/String;Ljava/lang/String;J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngine_restoreDbFromLatestBackup + (JNIEnv *, jobject, jlong, jstring, jstring, jlong); + +/* + * Class: org_forstdb_BackupEngine + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngine_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_BackupEngineOptions.h b/java/include/org_forstdb_BackupEngineOptions.h new file mode 100644 index 000000000..2368d6f56 --- /dev/null +++ b/java/include/org_forstdb_BackupEngineOptions.h @@ -0,0 +1,213 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_BackupEngineOptions */ + +#ifndef _Included_org_forstdb_BackupEngineOptions +#define _Included_org_forstdb_BackupEngineOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_BackupEngineOptions + * Method: newBackupEngineOptions + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_BackupEngineOptions_newBackupEngineOptions + (JNIEnv *, jclass, jstring); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: backupDir + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_BackupEngineOptions_backupDir + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setBackupEnv + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setBackupEnv + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setShareTableFiles + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setShareTableFiles + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: shareTableFiles + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_BackupEngineOptions_shareTableFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setInfoLog + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setInfoLog + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setSync + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setSync + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: sync + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_BackupEngineOptions_sync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setDestroyOldData + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setDestroyOldData + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: destroyOldData + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_BackupEngineOptions_destroyOldData + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setBackupLogFiles + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setBackupLogFiles + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: backupLogFiles + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_BackupEngineOptions_backupLogFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setBackupRateLimit + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setBackupRateLimit + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: backupRateLimit + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_BackupEngineOptions_backupRateLimit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setBackupRateLimiter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setBackupRateLimiter + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setRestoreRateLimit + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setRestoreRateLimit + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: restoreRateLimit + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_BackupEngineOptions_restoreRateLimit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setRestoreRateLimiter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setRestoreRateLimiter + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setShareFilesWithChecksum + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setShareFilesWithChecksum + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: shareFilesWithChecksum + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_BackupEngineOptions_shareFilesWithChecksum + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setMaxBackgroundOperations + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setMaxBackgroundOperations + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: maxBackgroundOperations + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_BackupEngineOptions_maxBackgroundOperations + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: setCallbackTriggerIntervalSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_setCallbackTriggerIntervalSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: callbackTriggerIntervalSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_BackupEngineOptions_callbackTriggerIntervalSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_BackupEngineOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_BackupEngineOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_BlockBasedTableConfig.h b/java/include/org_forstdb_BlockBasedTableConfig.h new file mode 100644 index 000000000..b83bdf655 --- /dev/null +++ b/java/include/org_forstdb_BlockBasedTableConfig.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_BlockBasedTableConfig */ + +#ifndef _Included_org_forstdb_BlockBasedTableConfig +#define _Included_org_forstdb_BlockBasedTableConfig +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_BlockBasedTableConfig + * Method: newTableFactoryHandle + * Signature: (ZZZZBBDBZJJJIIIJZZZJZZIIZZBJI)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_BlockBasedTableConfig_newTableFactoryHandle + (JNIEnv *, jobject, jboolean, jboolean, jboolean, jboolean, jbyte, jbyte, jdouble, jbyte, jboolean, jlong, jlong, jlong, jint, jint, jint, jlong, jboolean, jboolean, jboolean, jlong, jboolean, jboolean, jint, jint, jboolean, jboolean, jbyte, jlong, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_BloomFilter.h b/java/include/org_forstdb_BloomFilter.h new file mode 100644 index 000000000..95d43d194 --- /dev/null +++ b/java/include/org_forstdb_BloomFilter.h @@ -0,0 +1,23 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_BloomFilter */ + +#ifndef _Included_org_forstdb_BloomFilter +#define _Included_org_forstdb_BloomFilter +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_BloomFilter_DEFAULT_BITS_PER_KEY +#define org_forstdb_BloomFilter_DEFAULT_BITS_PER_KEY 10.0 +/* + * Class: org_forstdb_BloomFilter + * Method: createNewBloomFilter + * Signature: (D)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_BloomFilter_createNewBloomFilter + (JNIEnv *, jclass, jdouble); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Cache.h b/java/include/org_forstdb_Cache.h new file mode 100644 index 000000000..219d121ad --- /dev/null +++ b/java/include/org_forstdb_Cache.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Cache */ + +#ifndef _Included_org_forstdb_Cache +#define _Included_org_forstdb_Cache +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Cache + * Method: getUsage + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Cache_getUsage + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Cache + * Method: getPinnedUsage + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Cache_getPinnedUsage + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CassandraCompactionFilter.h b/java/include/org_forstdb_CassandraCompactionFilter.h new file mode 100644 index 000000000..76c66b9e7 --- /dev/null +++ b/java/include/org_forstdb_CassandraCompactionFilter.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CassandraCompactionFilter */ + +#ifndef _Included_org_forstdb_CassandraCompactionFilter +#define _Included_org_forstdb_CassandraCompactionFilter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_CassandraCompactionFilter + * Method: createNewCassandraCompactionFilter0 + * Signature: (ZI)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CassandraCompactionFilter_createNewCassandraCompactionFilter0 + (JNIEnv *, jclass, jboolean, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CassandraValueMergeOperator.h b/java/include/org_forstdb_CassandraValueMergeOperator.h new file mode 100644 index 000000000..a467d52cc --- /dev/null +++ b/java/include/org_forstdb_CassandraValueMergeOperator.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CassandraValueMergeOperator */ + +#ifndef _Included_org_forstdb_CassandraValueMergeOperator +#define _Included_org_forstdb_CassandraValueMergeOperator +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_CassandraValueMergeOperator + * Method: newSharedCassandraValueMergeOperator + * Signature: (II)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CassandraValueMergeOperator_newSharedCassandraValueMergeOperator + (JNIEnv *, jclass, jint, jint); + +/* + * Class: org_forstdb_CassandraValueMergeOperator + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CassandraValueMergeOperator_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Checkpoint.h b/java/include/org_forstdb_Checkpoint.h new file mode 100644 index 000000000..59021737c --- /dev/null +++ b/java/include/org_forstdb_Checkpoint.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Checkpoint */ + +#ifndef _Included_org_forstdb_Checkpoint +#define _Included_org_forstdb_Checkpoint +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Checkpoint + * Method: newCheckpoint + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Checkpoint_newCheckpoint + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Checkpoint + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Checkpoint_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Checkpoint + * Method: createCheckpoint + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Checkpoint_createCheckpoint + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_Checkpoint + * Method: exportColumnFamily + * Signature: (JJLjava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Checkpoint_exportColumnFamily + (JNIEnv *, jobject, jlong, jlong, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ClockCache.h b/java/include/org_forstdb_ClockCache.h new file mode 100644 index 000000000..24533d053 --- /dev/null +++ b/java/include/org_forstdb_ClockCache.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ClockCache */ + +#ifndef _Included_org_forstdb_ClockCache +#define _Included_org_forstdb_ClockCache +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ClockCache + * Method: newClockCache + * Signature: (JIZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ClockCache_newClockCache + (JNIEnv *, jclass, jlong, jint, jboolean); + +/* + * Class: org_forstdb_ClockCache + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ClockCache_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ColumnFamilyHandle.h b/java/include/org_forstdb_ColumnFamilyHandle.h new file mode 100644 index 000000000..d14687dbe --- /dev/null +++ b/java/include/org_forstdb_ColumnFamilyHandle.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ColumnFamilyHandle */ + +#ifndef _Included_org_forstdb_ColumnFamilyHandle +#define _Included_org_forstdb_ColumnFamilyHandle +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ColumnFamilyHandle + * Method: getName + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_ColumnFamilyHandle_getName + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyHandle + * Method: getID + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyHandle_getID + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyHandle + * Method: getDescriptor + * Signature: (J)Lorg/forstdb/ColumnFamilyDescriptor; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_ColumnFamilyHandle_getDescriptor + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyHandle + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyHandle_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ColumnFamilyOptions.h b/java/include/org_forstdb_ColumnFamilyOptions.h new file mode 100644 index 000000000..0e4e7c3e2 --- /dev/null +++ b/java/include/org_forstdb_ColumnFamilyOptions.h @@ -0,0 +1,1141 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ColumnFamilyOptions */ + +#ifndef _Included_org_forstdb_ColumnFamilyOptions +#define _Included_org_forstdb_ColumnFamilyOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: getColumnFamilyOptionsFromProps + * Signature: (JLjava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__JLjava_lang_String_2 + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: getColumnFamilyOptionsFromProps + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__Ljava_lang_String_2 + (JNIEnv *, jclass, jstring); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: newColumnFamilyOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_newColumnFamilyOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: copyColumnFamilyOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_copyColumnFamilyOptions + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: newColumnFamilyOptionsFromOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_newColumnFamilyOptionsFromOptions + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: oldDefaults + * Signature: (JII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_oldDefaults + (JNIEnv *, jclass, jlong, jint, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: optimizeForSmallDb + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_optimizeForSmallDb__J + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: optimizeForSmallDb + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_optimizeForSmallDb__JJ + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: optimizeForPointLookup + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_optimizeForPointLookup + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: optimizeLevelStyleCompaction + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_optimizeLevelStyleCompaction + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: optimizeUniversalStyleCompaction + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_optimizeUniversalStyleCompaction + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setComparatorHandle + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setComparatorHandle__JI + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setComparatorHandle + * Signature: (JJB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setComparatorHandle__JJB + (JNIEnv *, jobject, jlong, jlong, jbyte); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMergeOperatorName + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMergeOperatorName + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMergeOperator + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMergeOperator + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompactionFilterHandle + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompactionFilterHandle + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompactionFilterFactoryHandle + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompactionFilterFactoryHandle + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setWriteBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setWriteBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: writeBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_writeBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxWriteBufferNumber + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxWriteBufferNumber + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxWriteBufferNumber + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_maxWriteBufferNumber + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMinWriteBufferNumberToMerge + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMinWriteBufferNumberToMerge + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: minWriteBufferNumberToMerge + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_minWriteBufferNumberToMerge + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompressionType + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompressionType + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: compressionType + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_ColumnFamilyOptions_compressionType + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompressionPerLevel + * Signature: (J[B)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompressionPerLevel + (JNIEnv *, jobject, jlong, jbyteArray); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: compressionPerLevel + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_ColumnFamilyOptions_compressionPerLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBottommostCompressionType + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBottommostCompressionType + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: bottommostCompressionType + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_ColumnFamilyOptions_bottommostCompressionType + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBottommostCompressionOptions + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBottommostCompressionOptions + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompressionOptions + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompressionOptions + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: useFixedLengthPrefixExtractor + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_useFixedLengthPrefixExtractor + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: useCappedPrefixExtractor + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_useCappedPrefixExtractor + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setNumLevels + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setNumLevels + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: numLevels + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_numLevels + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setLevelZeroFileNumCompactionTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setLevelZeroFileNumCompactionTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: levelZeroFileNumCompactionTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_levelZeroFileNumCompactionTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setLevelZeroSlowdownWritesTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setLevelZeroSlowdownWritesTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: levelZeroSlowdownWritesTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_levelZeroSlowdownWritesTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setLevelZeroStopWritesTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setLevelZeroStopWritesTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: levelZeroStopWritesTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_levelZeroStopWritesTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setTargetFileSizeBase + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setTargetFileSizeBase + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: targetFileSizeBase + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_targetFileSizeBase + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setTargetFileSizeMultiplier + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setTargetFileSizeMultiplier + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: targetFileSizeMultiplier + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_targetFileSizeMultiplier + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxBytesForLevelBase + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxBytesForLevelBase + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxBytesForLevelBase + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_maxBytesForLevelBase + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setLevelCompactionDynamicLevelBytes + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setLevelCompactionDynamicLevelBytes + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: levelCompactionDynamicLevelBytes + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_levelCompactionDynamicLevelBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxBytesForLevelMultiplier + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplier + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxBytesForLevelMultiplier + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_ColumnFamilyOptions_maxBytesForLevelMultiplier + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxCompactionBytes + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxCompactionBytes + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxCompactionBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_maxCompactionBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setArenaBlockSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setArenaBlockSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: arenaBlockSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_arenaBlockSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setDisableAutoCompactions + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setDisableAutoCompactions + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: disableAutoCompactions + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_disableAutoCompactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompactionStyle + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompactionStyle + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: compactionStyle + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_ColumnFamilyOptions_compactionStyle + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxTableFilesSizeFIFO + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxTableFilesSizeFIFO + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxTableFilesSizeFIFO + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_maxTableFilesSizeFIFO + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxSequentialSkipInIterations + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxSequentialSkipInIterations + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxSequentialSkipInIterations + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_maxSequentialSkipInIterations + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMemTableFactory + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMemTableFactory + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: memTableFactoryName + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ColumnFamilyOptions_memTableFactoryName + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setTableFactory + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setTableFactory + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: tableFactoryName + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ColumnFamilyOptions_tableFactoryName + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCfPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCfPaths + (JNIEnv *, jclass, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: cfPathsLen + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_cfPathsLen + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: cfPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_cfPaths + (JNIEnv *, jclass, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setInplaceUpdateSupport + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setInplaceUpdateSupport + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: inplaceUpdateSupport + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_inplaceUpdateSupport + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setInplaceUpdateNumLocks + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setInplaceUpdateNumLocks + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: inplaceUpdateNumLocks + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_inplaceUpdateNumLocks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMemtablePrefixBloomSizeRatio + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: memtablePrefixBloomSizeRatio + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_ColumnFamilyOptions_memtablePrefixBloomSizeRatio + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setExperimentalMempurgeThreshold + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setExperimentalMempurgeThreshold + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: experimentalMempurgeThreshold + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_ColumnFamilyOptions_experimentalMempurgeThreshold + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMemtableWholeKeyFiltering + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: memtableWholeKeyFiltering + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_memtableWholeKeyFiltering + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBloomLocality + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBloomLocality + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: bloomLocality + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_bloomLocality + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxSuccessiveMerges + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxSuccessiveMerges + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxSuccessiveMerges + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_maxSuccessiveMerges + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setOptimizeFiltersForHits + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setOptimizeFiltersForHits + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: optimizeFiltersForHits + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_optimizeFiltersForHits + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMemtableHugePageSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMemtableHugePageSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: memtableHugePageSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_memtableHugePageSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setSoftPendingCompactionBytesLimit + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setSoftPendingCompactionBytesLimit + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: softPendingCompactionBytesLimit + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_softPendingCompactionBytesLimit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setHardPendingCompactionBytesLimit + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setHardPendingCompactionBytesLimit + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: hardPendingCompactionBytesLimit + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_hardPendingCompactionBytesLimit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setLevel0FileNumCompactionTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setLevel0FileNumCompactionTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: level0FileNumCompactionTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_level0FileNumCompactionTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setLevel0SlowdownWritesTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setLevel0SlowdownWritesTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: level0SlowdownWritesTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_level0SlowdownWritesTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setLevel0StopWritesTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setLevel0StopWritesTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: level0StopWritesTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_level0StopWritesTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxBytesForLevelMultiplierAdditional + * Signature: (J[I)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplierAdditional + (JNIEnv *, jobject, jlong, jintArray); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxBytesForLevelMultiplierAdditional + * Signature: (J)[I + */ +JNIEXPORT jintArray JNICALL Java_org_forstdb_ColumnFamilyOptions_maxBytesForLevelMultiplierAdditional + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setParanoidFileChecks + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setParanoidFileChecks + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: paranoidFileChecks + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_paranoidFileChecks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMaxWriteBufferNumberToMaintain + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMaxWriteBufferNumberToMaintain + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: maxWriteBufferNumberToMaintain + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_maxWriteBufferNumberToMaintain + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompactionPriority + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompactionPriority + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: compactionPriority + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_ColumnFamilyOptions_compactionPriority + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setReportBgIoStats + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setReportBgIoStats + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: reportBgIoStats + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_reportBgIoStats + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setTtl + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setTtl + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: ttl + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_ttl + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setPeriodicCompactionSeconds + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setPeriodicCompactionSeconds + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: periodicCompactionSeconds + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_periodicCompactionSeconds + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompactionOptionsUniversal + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompactionOptionsUniversal + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompactionOptionsFIFO + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompactionOptionsFIFO + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setForceConsistencyChecks + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setForceConsistencyChecks + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: forceConsistencyChecks + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_forceConsistencyChecks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setSstPartitionerFactory + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setSstPartitionerFactory + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setCompactionThreadLimiter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setCompactionThreadLimiter + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMemtableMaxRangeDeletions + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMemtableMaxRangeDeletions + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: memtableMaxRangeDeletions + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_memtableMaxRangeDeletions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setEnableBlobFiles + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setEnableBlobFiles + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: enableBlobFiles + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_enableBlobFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setMinBlobSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setMinBlobSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: minBlobSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_minBlobSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBlobFileSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBlobFileSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: blobFileSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_blobFileSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBlobCompressionType + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBlobCompressionType + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: blobCompressionType + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_ColumnFamilyOptions_blobCompressionType + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setEnableBlobGarbageCollection + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setEnableBlobGarbageCollection + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: enableBlobGarbageCollection + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ColumnFamilyOptions_enableBlobGarbageCollection + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBlobGarbageCollectionAgeCutoff + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBlobGarbageCollectionAgeCutoff + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: blobGarbageCollectionAgeCutoff + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_ColumnFamilyOptions_blobGarbageCollectionAgeCutoff + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBlobGarbageCollectionForceThreshold + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: blobGarbageCollectionForceThreshold + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBlobCompactionReadaheadSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBlobCompactionReadaheadSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: blobCompactionReadaheadSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ColumnFamilyOptions_blobCompactionReadaheadSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setBlobFileStartingLevel + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setBlobFileStartingLevel + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: blobFileStartingLevel + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ColumnFamilyOptions_blobFileStartingLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: setPrepopulateBlobCache + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ColumnFamilyOptions_setPrepopulateBlobCache + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_ColumnFamilyOptions + * Method: prepopulateBlobCache + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_ColumnFamilyOptions_prepopulateBlobCache + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CompactRangeOptions.h b/java/include/org_forstdb_CompactRangeOptions.h new file mode 100644 index 000000000..40b48a147 --- /dev/null +++ b/java/include/org_forstdb_CompactRangeOptions.h @@ -0,0 +1,181 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CompactRangeOptions */ + +#ifndef _Included_org_forstdb_CompactRangeOptions +#define _Included_org_forstdb_CompactRangeOptions +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_CompactRangeOptions_VALUE_kSkip +#define org_forstdb_CompactRangeOptions_VALUE_kSkip 0L +#undef org_forstdb_CompactRangeOptions_VALUE_kIfHaveCompactionFilter +#define org_forstdb_CompactRangeOptions_VALUE_kIfHaveCompactionFilter 1L +#undef org_forstdb_CompactRangeOptions_VALUE_kForce +#define org_forstdb_CompactRangeOptions_VALUE_kForce 2L +#undef org_forstdb_CompactRangeOptions_VALUE_kForceOptimized +#define org_forstdb_CompactRangeOptions_VALUE_kForceOptimized 3L +/* + * Class: org_forstdb_CompactRangeOptions + * Method: newCompactRangeOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactRangeOptions_newCompactRangeOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: exclusiveManualCompaction + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_CompactRangeOptions_exclusiveManualCompaction + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setExclusiveManualCompaction + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setExclusiveManualCompaction + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: changeLevel + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_CompactRangeOptions_changeLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setChangeLevel + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setChangeLevel + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: targetLevel + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactRangeOptions_targetLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setTargetLevel + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setTargetLevel + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: targetPathId + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactRangeOptions_targetPathId + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setTargetPathId + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setTargetPathId + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: bottommostLevelCompaction + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactRangeOptions_bottommostLevelCompaction + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setBottommostLevelCompaction + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setBottommostLevelCompaction + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: allowWriteStall + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_CompactRangeOptions_allowWriteStall + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setAllowWriteStall + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setAllowWriteStall + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setMaxSubcompactions + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setMaxSubcompactions + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: maxSubcompactions + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactRangeOptions_maxSubcompactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setFullHistoryTSLow + * Signature: (JJJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setFullHistoryTSLow + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: fullHistoryTSLow + * Signature: (J)Lorg/forstdb/CompactRangeOptions/Timestamp; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_CompactRangeOptions_fullHistoryTSLow + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: setCanceled + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactRangeOptions_setCanceled + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_CompactRangeOptions + * Method: canceled + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_CompactRangeOptions_canceled + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CompactionJobInfo.h b/java/include/org_forstdb_CompactionJobInfo.h new file mode 100644 index 000000000..35122098e --- /dev/null +++ b/java/include/org_forstdb_CompactionJobInfo.h @@ -0,0 +1,125 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CompactionJobInfo */ + +#ifndef _Included_org_forstdb_CompactionJobInfo +#define _Included_org_forstdb_CompactionJobInfo +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_CompactionJobInfo + * Method: newCompactionJobInfo + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobInfo_newCompactionJobInfo + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionJobInfo_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: columnFamilyName + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_CompactionJobInfo_columnFamilyName + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: status + * Signature: (J)Lorg/forstdb/Status; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_CompactionJobInfo_status + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: threadId + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobInfo_threadId + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: jobId + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionJobInfo_jobId + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: baseInputLevel + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionJobInfo_baseInputLevel + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: outputLevel + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionJobInfo_outputLevel + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: inputFiles + * Signature: (J)[Ljava/lang/String; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_CompactionJobInfo_inputFiles + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: outputFiles + * Signature: (J)[Ljava/lang/String; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_CompactionJobInfo_outputFiles + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: tableProperties + * Signature: (J)Ljava/util/Map; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_CompactionJobInfo_tableProperties + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: compactionReason + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_CompactionJobInfo_compactionReason + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: compression + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_CompactionJobInfo_compression + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobInfo + * Method: stats + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobInfo_stats + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CompactionJobStats.h b/java/include/org_forstdb_CompactionJobStats.h new file mode 100644 index 000000000..5bdb2ec33 --- /dev/null +++ b/java/include/org_forstdb_CompactionJobStats.h @@ -0,0 +1,229 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CompactionJobStats */ + +#ifndef _Included_org_forstdb_CompactionJobStats +#define _Included_org_forstdb_CompactionJobStats +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_CompactionJobStats + * Method: newCompactionJobStats + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_newCompactionJobStats + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionJobStats_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: reset + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionJobStats_reset + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: add + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionJobStats_add + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: elapsedMicros + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_elapsedMicros + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numInputRecords + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numInputRecords + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numInputFiles + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numInputFiles + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numInputFilesAtOutputLevel + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numInputFilesAtOutputLevel + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numOutputRecords + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numOutputRecords + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numOutputFiles + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numOutputFiles + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: isManualCompaction + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_CompactionJobStats_isManualCompaction + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: totalInputBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_totalInputBytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: totalOutputBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_totalOutputBytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numRecordsReplaced + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numRecordsReplaced + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: totalInputRawKeyBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_totalInputRawKeyBytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: totalInputRawValueBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_totalInputRawValueBytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numInputDeletionRecords + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numInputDeletionRecords + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numExpiredDeletionRecords + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numExpiredDeletionRecords + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numCorruptKeys + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numCorruptKeys + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: fileWriteNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_fileWriteNanos + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: fileRangeSyncNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_fileRangeSyncNanos + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: fileFsyncNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_fileFsyncNanos + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: filePrepareWriteNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_filePrepareWriteNanos + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: smallestOutputKeyPrefix + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_CompactionJobStats_smallestOutputKeyPrefix + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: largestOutputKeyPrefix + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_CompactionJobStats_largestOutputKeyPrefix + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numSingleDelFallthru + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numSingleDelFallthru + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionJobStats + * Method: numSingleDelMismatch + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionJobStats_numSingleDelMismatch + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CompactionOptions.h b/java/include/org_forstdb_CompactionOptions.h new file mode 100644 index 000000000..9de502251 --- /dev/null +++ b/java/include/org_forstdb_CompactionOptions.h @@ -0,0 +1,77 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CompactionOptions */ + +#ifndef _Included_org_forstdb_CompactionOptions +#define _Included_org_forstdb_CompactionOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_CompactionOptions + * Method: newCompactionOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionOptions_newCompactionOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_CompactionOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptions + * Method: compression + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_CompactionOptions_compression + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionOptions + * Method: setCompression + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptions_setCompression + (JNIEnv *, jclass, jlong, jbyte); + +/* + * Class: org_forstdb_CompactionOptions + * Method: outputFileSizeLimit + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionOptions_outputFileSizeLimit + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionOptions + * Method: setOutputFileSizeLimit + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptions_setOutputFileSizeLimit + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_CompactionOptions + * Method: maxSubcompactions + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionOptions_maxSubcompactions + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_CompactionOptions + * Method: setMaxSubcompactions + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptions_setMaxSubcompactions + (JNIEnv *, jclass, jlong, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CompactionOptionsFIFO.h b/java/include/org_forstdb_CompactionOptionsFIFO.h new file mode 100644 index 000000000..aed1c4b69 --- /dev/null +++ b/java/include/org_forstdb_CompactionOptionsFIFO.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CompactionOptionsFIFO */ + +#ifndef _Included_org_forstdb_CompactionOptionsFIFO +#define _Included_org_forstdb_CompactionOptionsFIFO +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_CompactionOptionsFIFO + * Method: newCompactionOptionsFIFO + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionOptionsFIFO_newCompactionOptionsFIFO + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_CompactionOptionsFIFO + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsFIFO_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsFIFO + * Method: setMaxTableFilesSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsFIFO_setMaxTableFilesSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_CompactionOptionsFIFO + * Method: maxTableFilesSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionOptionsFIFO_maxTableFilesSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsFIFO + * Method: setAllowCompaction + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsFIFO_setAllowCompaction + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_CompactionOptionsFIFO + * Method: allowCompaction + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_CompactionOptionsFIFO_allowCompaction + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CompactionOptionsUniversal.h b/java/include/org_forstdb_CompactionOptionsUniversal.h new file mode 100644 index 000000000..606032f24 --- /dev/null +++ b/java/include/org_forstdb_CompactionOptionsUniversal.h @@ -0,0 +1,141 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CompactionOptionsUniversal */ + +#ifndef _Included_org_forstdb_CompactionOptionsUniversal +#define _Included_org_forstdb_CompactionOptionsUniversal +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: newCompactionOptionsUniversal + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompactionOptionsUniversal_newCompactionOptionsUniversal + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsUniversal_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: setSizeRatio + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsUniversal_setSizeRatio + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: sizeRatio + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionOptionsUniversal_sizeRatio + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: setMinMergeWidth + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsUniversal_setMinMergeWidth + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: minMergeWidth + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionOptionsUniversal_minMergeWidth + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: setMaxMergeWidth + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsUniversal_setMaxMergeWidth + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: maxMergeWidth + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionOptionsUniversal_maxMergeWidth + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: setMaxSizeAmplificationPercent + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsUniversal_setMaxSizeAmplificationPercent + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: maxSizeAmplificationPercent + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionOptionsUniversal_maxSizeAmplificationPercent + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: setCompressionSizePercent + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsUniversal_setCompressionSizePercent + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: compressionSizePercent + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompactionOptionsUniversal_compressionSizePercent + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: setStopStyle + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsUniversal_setStopStyle + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: stopStyle + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_CompactionOptionsUniversal_stopStyle + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: setAllowTrivialMove + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompactionOptionsUniversal_setAllowTrivialMove + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_CompactionOptionsUniversal + * Method: allowTrivialMove + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_CompactionOptionsUniversal_allowTrivialMove + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ComparatorOptions.h b/java/include/org_forstdb_ComparatorOptions.h new file mode 100644 index 000000000..68c0846ea --- /dev/null +++ b/java/include/org_forstdb_ComparatorOptions.h @@ -0,0 +1,77 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ComparatorOptions */ + +#ifndef _Included_org_forstdb_ComparatorOptions +#define _Included_org_forstdb_ComparatorOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ComparatorOptions + * Method: newComparatorOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ComparatorOptions_newComparatorOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_ComparatorOptions + * Method: reusedSynchronisationType + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_ComparatorOptions_reusedSynchronisationType + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ComparatorOptions + * Method: setReusedSynchronisationType + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ComparatorOptions_setReusedSynchronisationType + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_ComparatorOptions + * Method: useDirectBuffer + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ComparatorOptions_useDirectBuffer + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ComparatorOptions + * Method: setUseDirectBuffer + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ComparatorOptions_setUseDirectBuffer + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ComparatorOptions + * Method: maxReusedBufferSize + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ComparatorOptions_maxReusedBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ComparatorOptions + * Method: setMaxReusedBufferSize + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ComparatorOptions_setMaxReusedBufferSize + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_ComparatorOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ComparatorOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_CompressionOptions.h b/java/include/org_forstdb_CompressionOptions.h new file mode 100644 index 000000000..b5d7fc79b --- /dev/null +++ b/java/include/org_forstdb_CompressionOptions.h @@ -0,0 +1,125 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_CompressionOptions */ + +#ifndef _Included_org_forstdb_CompressionOptions +#define _Included_org_forstdb_CompressionOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_CompressionOptions + * Method: newCompressionOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_CompressionOptions_newCompressionOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_CompressionOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompressionOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompressionOptions + * Method: setWindowBits + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompressionOptions_setWindowBits + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompressionOptions + * Method: windowBits + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompressionOptions_windowBits + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompressionOptions + * Method: setLevel + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompressionOptions_setLevel + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompressionOptions + * Method: level + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompressionOptions_level + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompressionOptions + * Method: setStrategy + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompressionOptions_setStrategy + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompressionOptions + * Method: strategy + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompressionOptions_strategy + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompressionOptions + * Method: setMaxDictBytes + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompressionOptions_setMaxDictBytes + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompressionOptions + * Method: maxDictBytes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompressionOptions_maxDictBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompressionOptions + * Method: setZstdMaxTrainBytes + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompressionOptions_setZstdMaxTrainBytes + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_CompressionOptions + * Method: zstdMaxTrainBytes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_CompressionOptions_zstdMaxTrainBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_CompressionOptions + * Method: setEnabled + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_CompressionOptions_setEnabled + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_CompressionOptions + * Method: enabled + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_CompressionOptions_enabled + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ConcurrentTaskLimiterImpl.h b/java/include/org_forstdb_ConcurrentTaskLimiterImpl.h new file mode 100644 index 000000000..e8ae61f40 --- /dev/null +++ b/java/include/org_forstdb_ConcurrentTaskLimiterImpl.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ConcurrentTaskLimiterImpl */ + +#ifndef _Included_org_forstdb_ConcurrentTaskLimiterImpl +#define _Included_org_forstdb_ConcurrentTaskLimiterImpl +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ConcurrentTaskLimiterImpl + * Method: newConcurrentTaskLimiterImpl0 + * Signature: (Ljava/lang/String;I)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ConcurrentTaskLimiterImpl_newConcurrentTaskLimiterImpl0 + (JNIEnv *, jclass, jstring, jint); + +/* + * Class: org_forstdb_ConcurrentTaskLimiterImpl + * Method: name + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ConcurrentTaskLimiterImpl_name + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_ConcurrentTaskLimiterImpl + * Method: setMaxOutstandingTask + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConcurrentTaskLimiterImpl_setMaxOutstandingTask + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_forstdb_ConcurrentTaskLimiterImpl + * Method: resetMaxOutstandingTask + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConcurrentTaskLimiterImpl_resetMaxOutstandingTask + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_ConcurrentTaskLimiterImpl + * Method: outstandingTask + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_ConcurrentTaskLimiterImpl_outstandingTask + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_ConcurrentTaskLimiterImpl + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConcurrentTaskLimiterImpl_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ConfigOptions.h b/java/include/org_forstdb_ConfigOptions.h new file mode 100644 index 000000000..cd3afd215 --- /dev/null +++ b/java/include/org_forstdb_ConfigOptions.h @@ -0,0 +1,69 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ConfigOptions */ + +#ifndef _Included_org_forstdb_ConfigOptions +#define _Included_org_forstdb_ConfigOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ConfigOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConfigOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ConfigOptions + * Method: newConfigOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ConfigOptions_newConfigOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_ConfigOptions + * Method: setEnv + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConfigOptions_setEnv + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_ConfigOptions + * Method: setDelimiter + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConfigOptions_setDelimiter + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: org_forstdb_ConfigOptions + * Method: setIgnoreUnknownOptions + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConfigOptions_setIgnoreUnknownOptions + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_ConfigOptions + * Method: setInputStringsEscaped + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConfigOptions_setInputStringsEscaped + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_ConfigOptions + * Method: setSanityLevel + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ConfigOptions_setSanityLevel + (JNIEnv *, jclass, jlong, jbyte); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_DBOptions.h b/java/include/org_forstdb_DBOptions.h new file mode 100644 index 000000000..1392c0c3d --- /dev/null +++ b/java/include/org_forstdb_DBOptions.h @@ -0,0 +1,1343 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_DBOptions */ + +#ifndef _Included_org_forstdb_DBOptions +#define _Included_org_forstdb_DBOptions +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_DBOptions_DEFAULT_NUM_SHARD_BITS +#define org_forstdb_DBOptions_DEFAULT_NUM_SHARD_BITS -1L +/* + * Class: org_forstdb_DBOptions + * Method: getDBOptionsFromProps + * Signature: (JLjava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_getDBOptionsFromProps__JLjava_lang_String_2 + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: org_forstdb_DBOptions + * Method: getDBOptionsFromProps + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_getDBOptionsFromProps__Ljava_lang_String_2 + (JNIEnv *, jclass, jstring); + +/* + * Class: org_forstdb_DBOptions + * Method: newDBOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_newDBOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_DBOptions + * Method: copyDBOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_copyDBOptions + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: newDBOptionsFromOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_newDBOptionsFromOptions + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: optimizeForSmallDb + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_optimizeForSmallDb + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setIncreaseParallelism + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setIncreaseParallelism + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: setCreateIfMissing + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setCreateIfMissing + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: createIfMissing + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_createIfMissing + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setCreateMissingColumnFamilies + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setCreateMissingColumnFamilies + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: createMissingColumnFamilies + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_createMissingColumnFamilies + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setEnv + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setEnv + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setErrorIfExists + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setErrorIfExists + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: errorIfExists + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_errorIfExists + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setParanoidChecks + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setParanoidChecks + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: paranoidChecks + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_paranoidChecks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setRateLimiter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setRateLimiter + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setSstFileManager + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setSstFileManager + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setLogger + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setLogger + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setInfoLogLevel + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setInfoLogLevel + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_DBOptions + * Method: infoLogLevel + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_DBOptions_infoLogLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxOpenFiles + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxOpenFiles + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: maxOpenFiles + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_maxOpenFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxFileOpeningThreads + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxFileOpeningThreads + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: maxFileOpeningThreads + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_maxFileOpeningThreads + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxTotalWalSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxTotalWalSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: maxTotalWalSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_maxTotalWalSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setStatistics + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setStatistics + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: statistics + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_statistics + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: useFsync + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_useFsync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setUseFsync + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setUseFsync + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: setDbPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setDbPaths + (JNIEnv *, jobject, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_DBOptions + * Method: dbPathsLen + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_dbPathsLen + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: dbPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_dbPaths + (JNIEnv *, jobject, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_DBOptions + * Method: setDbLogDir + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setDbLogDir + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_DBOptions + * Method: dbLogDir + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_DBOptions_dbLogDir + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWalDir + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWalDir + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_DBOptions + * Method: walDir + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_DBOptions_walDir + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setDeleteObsoleteFilesPeriodMicros + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setDeleteObsoleteFilesPeriodMicros + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: deleteObsoleteFilesPeriodMicros + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_deleteObsoleteFilesPeriodMicros + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxBackgroundCompactions + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxBackgroundCompactions + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: maxBackgroundCompactions + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_maxBackgroundCompactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxSubcompactions + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxSubcompactions + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: maxSubcompactions + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_maxSubcompactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxBackgroundFlushes + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxBackgroundFlushes + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: maxBackgroundFlushes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_maxBackgroundFlushes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxBackgroundJobs + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxBackgroundJobs + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: maxBackgroundJobs + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_maxBackgroundJobs + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxLogFileSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxLogFileSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: maxLogFileSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_maxLogFileSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setLogFileTimeToRoll + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setLogFileTimeToRoll + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: logFileTimeToRoll + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_logFileTimeToRoll + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setKeepLogFileNum + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setKeepLogFileNum + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: keepLogFileNum + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_keepLogFileNum + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setRecycleLogFileNum + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setRecycleLogFileNum + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: recycleLogFileNum + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_recycleLogFileNum + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxManifestFileSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxManifestFileSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: maxManifestFileSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_maxManifestFileSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setTableCacheNumshardbits + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setTableCacheNumshardbits + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: tableCacheNumshardbits + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_tableCacheNumshardbits + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWalTtlSeconds + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWalTtlSeconds + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: walTtlSeconds + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_walTtlSeconds + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWalSizeLimitMB + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWalSizeLimitMB + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: walSizeLimitMB + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_walSizeLimitMB + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxWriteBatchGroupSizeBytes + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxWriteBatchGroupSizeBytes + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: maxWriteBatchGroupSizeBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_maxWriteBatchGroupSizeBytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setManifestPreallocationSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setManifestPreallocationSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: manifestPreallocationSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_manifestPreallocationSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setUseDirectReads + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setUseDirectReads + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: useDirectReads + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_useDirectReads + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setUseDirectIoForFlushAndCompaction + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setUseDirectIoForFlushAndCompaction + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: useDirectIoForFlushAndCompaction + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_useDirectIoForFlushAndCompaction + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAllowFAllocate + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAllowFAllocate + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: allowFAllocate + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_allowFAllocate + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAllowMmapReads + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAllowMmapReads + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: allowMmapReads + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_allowMmapReads + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAllowMmapWrites + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAllowMmapWrites + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: allowMmapWrites + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_allowMmapWrites + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setIsFdCloseOnExec + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setIsFdCloseOnExec + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: isFdCloseOnExec + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_isFdCloseOnExec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setStatsDumpPeriodSec + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setStatsDumpPeriodSec + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: statsDumpPeriodSec + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_statsDumpPeriodSec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setStatsPersistPeriodSec + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setStatsPersistPeriodSec + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: statsPersistPeriodSec + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_statsPersistPeriodSec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setStatsHistoryBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setStatsHistoryBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: statsHistoryBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_statsHistoryBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAdviseRandomOnOpen + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAdviseRandomOnOpen + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: adviseRandomOnOpen + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_adviseRandomOnOpen + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setDbWriteBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setDbWriteBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWriteBufferManager + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWriteBufferManager + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: dbWriteBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_dbWriteBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAccessHintOnCompactionStart + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAccessHintOnCompactionStart + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_DBOptions + * Method: accessHintOnCompactionStart + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_DBOptions_accessHintOnCompactionStart + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setCompactionReadaheadSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setCompactionReadaheadSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: compactionReadaheadSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_compactionReadaheadSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setRandomAccessMaxBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setRandomAccessMaxBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: randomAccessMaxBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_randomAccessMaxBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWritableFileMaxBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWritableFileMaxBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: writableFileMaxBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_writableFileMaxBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setUseAdaptiveMutex + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setUseAdaptiveMutex + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: useAdaptiveMutex + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_useAdaptiveMutex + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setBytesPerSync + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setBytesPerSync + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: bytesPerSync + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_bytesPerSync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWalBytesPerSync + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWalBytesPerSync + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: walBytesPerSync + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_walBytesPerSync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setStrictBytesPerSync + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setStrictBytesPerSync + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: strictBytesPerSync + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_strictBytesPerSync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setEventListeners + * Signature: (J[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setEventListeners + (JNIEnv *, jclass, jlong, jlongArray); + +/* + * Class: org_forstdb_DBOptions + * Method: eventListeners + * Signature: (J)[Lorg/forstdb/AbstractEventListener; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_DBOptions_eventListeners + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setEnableThreadTracking + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setEnableThreadTracking + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: enableThreadTracking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_enableThreadTracking + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setDelayedWriteRate + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setDelayedWriteRate + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: delayedWriteRate + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_delayedWriteRate + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setEnablePipelinedWrite + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setEnablePipelinedWrite + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: enablePipelinedWrite + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_enablePipelinedWrite + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setUnorderedWrite + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setUnorderedWrite + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: unorderedWrite + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_unorderedWrite + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAllowConcurrentMemtableWrite + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAllowConcurrentMemtableWrite + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: allowConcurrentMemtableWrite + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_allowConcurrentMemtableWrite + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setEnableWriteThreadAdaptiveYield + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setEnableWriteThreadAdaptiveYield + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: enableWriteThreadAdaptiveYield + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_enableWriteThreadAdaptiveYield + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWriteThreadMaxYieldUsec + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWriteThreadMaxYieldUsec + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: writeThreadMaxYieldUsec + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_writeThreadMaxYieldUsec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWriteThreadSlowYieldUsec + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWriteThreadSlowYieldUsec + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: writeThreadSlowYieldUsec + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_writeThreadSlowYieldUsec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setSkipStatsUpdateOnDbOpen + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setSkipStatsUpdateOnDbOpen + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: skipStatsUpdateOnDbOpen + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_skipStatsUpdateOnDbOpen + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setSkipCheckingSstFileSizesOnDbOpen + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setSkipCheckingSstFileSizesOnDbOpen + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: skipCheckingSstFileSizesOnDbOpen + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_skipCheckingSstFileSizesOnDbOpen + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWalRecoveryMode + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWalRecoveryMode + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_DBOptions + * Method: walRecoveryMode + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_DBOptions_walRecoveryMode + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAllow2pc + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAllow2pc + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: allow2pc + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_allow2pc + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setRowCache + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setRowCache + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWalFilter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWalFilter + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setFailIfOptionsFileError + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setFailIfOptionsFileError + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: failIfOptionsFileError + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_failIfOptionsFileError + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setDumpMallocStats + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setDumpMallocStats + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: dumpMallocStats + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_dumpMallocStats + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAvoidFlushDuringRecovery + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAvoidFlushDuringRecovery + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: avoidFlushDuringRecovery + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_avoidFlushDuringRecovery + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAvoidFlushDuringShutdown + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAvoidFlushDuringShutdown + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: avoidFlushDuringShutdown + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_avoidFlushDuringShutdown + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAllowIngestBehind + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAllowIngestBehind + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: allowIngestBehind + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_allowIngestBehind + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setTwoWriteQueues + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setTwoWriteQueues + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: twoWriteQueues + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_twoWriteQueues + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setManualWalFlush + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setManualWalFlush + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: manualWalFlush + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_manualWalFlush + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAtomicFlush + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAtomicFlush + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: atomicFlush + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_atomicFlush + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setAvoidUnnecessaryBlockingIO + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setAvoidUnnecessaryBlockingIO + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: avoidUnnecessaryBlockingIO + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_avoidUnnecessaryBlockingIO + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setPersistStatsToDisk + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setPersistStatsToDisk + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: persistStatsToDisk + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_persistStatsToDisk + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setWriteDbidToManifest + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setWriteDbidToManifest + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: writeDbidToManifest + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_writeDbidToManifest + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setLogReadaheadSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setLogReadaheadSize + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: logReadaheadSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_logReadaheadSize + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setBestEffortsRecovery + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setBestEffortsRecovery + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_DBOptions + * Method: bestEffortsRecovery + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_DBOptions_bestEffortsRecovery + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setMaxBgErrorResumeCount + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setMaxBgErrorResumeCount + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_forstdb_DBOptions + * Method: maxBgerrorResumeCount + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_DBOptions_maxBgerrorResumeCount + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: setBgerrorResumeRetryInterval + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DBOptions_setBgerrorResumeRetryInterval + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_DBOptions + * Method: bgerrorResumeRetryInterval + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DBOptions_bgerrorResumeRetryInterval + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_DirectSlice.h b/java/include/org_forstdb_DirectSlice.h new file mode 100644 index 000000000..ea809dcb9 --- /dev/null +++ b/java/include/org_forstdb_DirectSlice.h @@ -0,0 +1,77 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_DirectSlice */ + +#ifndef _Included_org_forstdb_DirectSlice +#define _Included_org_forstdb_DirectSlice +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_DirectSlice + * Method: createNewDirectSlice0 + * Signature: (Ljava/nio/ByteBuffer;I)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DirectSlice_createNewDirectSlice0 + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_forstdb_DirectSlice + * Method: createNewDirectSlice1 + * Signature: (Ljava/nio/ByteBuffer;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_DirectSlice_createNewDirectSlice1 + (JNIEnv *, jclass, jobject); + +/* + * Class: org_forstdb_DirectSlice + * Method: data0 + * Signature: (J)Ljava/nio/ByteBuffer; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_DirectSlice_data0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_DirectSlice + * Method: get0 + * Signature: (JI)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_DirectSlice_get0 + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DirectSlice + * Method: clear0 + * Signature: (JZJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DirectSlice_clear0 + (JNIEnv *, jobject, jlong, jboolean, jlong); + +/* + * Class: org_forstdb_DirectSlice + * Method: removePrefix0 + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DirectSlice_removePrefix0 + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DirectSlice + * Method: setLength0 + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DirectSlice_setLength0 + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_DirectSlice + * Method: disposeInternalBuf + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_DirectSlice_disposeInternalBuf + (JNIEnv *, jobject, jlong, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Env.h b/java/include/org_forstdb_Env.h new file mode 100644 index 000000000..8b9a95d66 --- /dev/null +++ b/java/include/org_forstdb_Env.h @@ -0,0 +1,77 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Env */ + +#ifndef _Included_org_forstdb_Env +#define _Included_org_forstdb_Env +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Env + * Method: getDefaultEnvInternal + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Env_getDefaultEnvInternal + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_Env + * Method: setBackgroundThreads + * Signature: (JIB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Env_setBackgroundThreads + (JNIEnv *, jobject, jlong, jint, jbyte); + +/* + * Class: org_forstdb_Env + * Method: getBackgroundThreads + * Signature: (JB)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Env_getBackgroundThreads + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Env + * Method: getThreadPoolQueueLen + * Signature: (JB)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Env_getThreadPoolQueueLen + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Env + * Method: incBackgroundThreadsIfNeeded + * Signature: (JIB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Env_incBackgroundThreadsIfNeeded + (JNIEnv *, jobject, jlong, jint, jbyte); + +/* + * Class: org_forstdb_Env + * Method: lowerThreadPoolIOPriority + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Env_lowerThreadPoolIOPriority + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Env + * Method: lowerThreadPoolCPUPriority + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Env_lowerThreadPoolCPUPriority + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Env + * Method: getThreadList + * Signature: (J)[Lorg/forstdb/ThreadStatus; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_Env_getThreadList + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_EnvFlinkTestSuite.h b/java/include/org_forstdb_EnvFlinkTestSuite.h new file mode 100644 index 000000000..1a880fa27 --- /dev/null +++ b/java/include/org_forstdb_EnvFlinkTestSuite.h @@ -0,0 +1,37 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_EnvFlinkTestSuite */ + +#ifndef _Included_org_forstdb_EnvFlinkTestSuite +#define _Included_org_forstdb_EnvFlinkTestSuite +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_EnvFlinkTestSuite + * Method: buildNativeObject + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_EnvFlinkTestSuite_buildNativeObject + (JNIEnv *, jobject, jstring); + +/* + * Class: org_forstdb_EnvFlinkTestSuite + * Method: runAllTestSuites + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvFlinkTestSuite_runAllTestSuites + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvFlinkTestSuite + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvFlinkTestSuite_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_EnvOptions.h b/java/include/org_forstdb_EnvOptions.h new file mode 100644 index 000000000..39795651a --- /dev/null +++ b/java/include/org_forstdb_EnvOptions.h @@ -0,0 +1,221 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_EnvOptions */ + +#ifndef _Included_org_forstdb_EnvOptions +#define _Included_org_forstdb_EnvOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_EnvOptions + * Method: newEnvOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_EnvOptions_newEnvOptions__ + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_EnvOptions + * Method: newEnvOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_EnvOptions_newEnvOptions__J + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setUseMmapReads + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setUseMmapReads + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_EnvOptions + * Method: useMmapReads + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_EnvOptions_useMmapReads + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setUseMmapWrites + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setUseMmapWrites + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_EnvOptions + * Method: useMmapWrites + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_EnvOptions_useMmapWrites + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setUseDirectReads + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setUseDirectReads + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_EnvOptions + * Method: useDirectReads + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_EnvOptions_useDirectReads + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setUseDirectWrites + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setUseDirectWrites + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_EnvOptions + * Method: useDirectWrites + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_EnvOptions_useDirectWrites + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setAllowFallocate + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setAllowFallocate + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_EnvOptions + * Method: allowFallocate + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_EnvOptions_allowFallocate + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setSetFdCloexec + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setSetFdCloexec + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_EnvOptions + * Method: setFdCloexec + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_EnvOptions_setFdCloexec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setBytesPerSync + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setBytesPerSync + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: bytesPerSync + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_EnvOptions_bytesPerSync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setFallocateWithKeepSize + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setFallocateWithKeepSize + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_EnvOptions + * Method: fallocateWithKeepSize + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_EnvOptions_fallocateWithKeepSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setCompactionReadaheadSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setCompactionReadaheadSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: compactionReadaheadSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_EnvOptions_compactionReadaheadSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setRandomAccessMaxBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setRandomAccessMaxBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: randomAccessMaxBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_EnvOptions_randomAccessMaxBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setWritableFileMaxBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setWritableFileMaxBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: writableFileMaxBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_EnvOptions_writableFileMaxBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_EnvOptions + * Method: setRateLimiter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_EnvOptions_setRateLimiter + (JNIEnv *, jobject, jlong, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ExportImportFilesMetaData.h b/java/include/org_forstdb_ExportImportFilesMetaData.h new file mode 100644 index 000000000..077daf31a --- /dev/null +++ b/java/include/org_forstdb_ExportImportFilesMetaData.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ExportImportFilesMetaData */ + +#ifndef _Included_org_forstdb_ExportImportFilesMetaData +#define _Included_org_forstdb_ExportImportFilesMetaData +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ExportImportFilesMetaData + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ExportImportFilesMetaData_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Filter.h b/java/include/org_forstdb_Filter.h new file mode 100644 index 000000000..948c5ecaa --- /dev/null +++ b/java/include/org_forstdb_Filter.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Filter */ + +#ifndef _Included_org_forstdb_Filter +#define _Included_org_forstdb_Filter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Filter + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Filter_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_FlinkCompactionFilter.h b/java/include/org_forstdb_FlinkCompactionFilter.h new file mode 100644 index 000000000..bb9bdb15c --- /dev/null +++ b/java/include/org_forstdb_FlinkCompactionFilter.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_FlinkCompactionFilter */ + +#ifndef _Included_org_forstdb_FlinkCompactionFilter +#define _Included_org_forstdb_FlinkCompactionFilter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_FlinkCompactionFilter + * Method: createNewFlinkCompactionFilter0 + * Signature: (JLorg/forstdb/FlinkCompactionFilter/TimeProvider;J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_FlinkCompactionFilter_createNewFlinkCompactionFilter0 + (JNIEnv *, jclass, jlong, jobject, jlong); + +/* + * Class: org_forstdb_FlinkCompactionFilter + * Method: createNewFlinkCompactionFilterConfigHolder + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_FlinkCompactionFilter_createNewFlinkCompactionFilterConfigHolder + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_FlinkCompactionFilter + * Method: disposeFlinkCompactionFilterConfigHolder + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_FlinkCompactionFilter_disposeFlinkCompactionFilterConfigHolder + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_FlinkCompactionFilter + * Method: configureFlinkCompactionFilter + * Signature: (JIIJJILorg/forstdb/FlinkCompactionFilter/ListElementFilterFactory;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_FlinkCompactionFilter_configureFlinkCompactionFilter + (JNIEnv *, jclass, jlong, jint, jint, jlong, jlong, jint, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_FlinkEnv.h b/java/include/org_forstdb_FlinkEnv.h new file mode 100644 index 000000000..4dfe9e786 --- /dev/null +++ b/java/include/org_forstdb_FlinkEnv.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_FlinkEnv */ + +#ifndef _Included_org_forstdb_FlinkEnv +#define _Included_org_forstdb_FlinkEnv +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_FlinkEnv + * Method: createFlinkEnv + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_FlinkEnv_createFlinkEnv + (JNIEnv *, jclass, jstring); + +/* + * Class: org_forstdb_FlinkEnv + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_FlinkEnv_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_FlushOptions.h b/java/include/org_forstdb_FlushOptions.h new file mode 100644 index 000000000..97ff71b99 --- /dev/null +++ b/java/include/org_forstdb_FlushOptions.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_FlushOptions */ + +#ifndef _Included_org_forstdb_FlushOptions +#define _Included_org_forstdb_FlushOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_FlushOptions + * Method: newFlushOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_FlushOptions_newFlushOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_FlushOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_FlushOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_FlushOptions + * Method: setWaitForFlush + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_FlushOptions_setWaitForFlush + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_FlushOptions + * Method: waitForFlush + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_FlushOptions_waitForFlush + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_FlushOptions + * Method: setAllowWriteStall + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_FlushOptions_setAllowWriteStall + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_FlushOptions + * Method: allowWriteStall + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_FlushOptions_allowWriteStall + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_HashLinkedListMemTableConfig.h b/java/include/org_forstdb_HashLinkedListMemTableConfig.h new file mode 100644 index 000000000..bfc29cab3 --- /dev/null +++ b/java/include/org_forstdb_HashLinkedListMemTableConfig.h @@ -0,0 +1,31 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_HashLinkedListMemTableConfig */ + +#ifndef _Included_org_forstdb_HashLinkedListMemTableConfig +#define _Included_org_forstdb_HashLinkedListMemTableConfig +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_HashLinkedListMemTableConfig_DEFAULT_BUCKET_COUNT +#define org_forstdb_HashLinkedListMemTableConfig_DEFAULT_BUCKET_COUNT 50000LL +#undef org_forstdb_HashLinkedListMemTableConfig_DEFAULT_HUGE_PAGE_TLB_SIZE +#define org_forstdb_HashLinkedListMemTableConfig_DEFAULT_HUGE_PAGE_TLB_SIZE 0LL +#undef org_forstdb_HashLinkedListMemTableConfig_DEFAULT_BUCKET_ENTRIES_LOG_THRES +#define org_forstdb_HashLinkedListMemTableConfig_DEFAULT_BUCKET_ENTRIES_LOG_THRES 4096L +#undef org_forstdb_HashLinkedListMemTableConfig_DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH +#define org_forstdb_HashLinkedListMemTableConfig_DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH 1L +#undef org_forstdb_HashLinkedListMemTableConfig_DEFAUL_THRESHOLD_USE_SKIPLIST +#define org_forstdb_HashLinkedListMemTableConfig_DEFAUL_THRESHOLD_USE_SKIPLIST 256L +/* + * Class: org_forstdb_HashLinkedListMemTableConfig + * Method: newMemTableFactoryHandle + * Signature: (JJIZI)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_HashLinkedListMemTableConfig_newMemTableFactoryHandle + (JNIEnv *, jobject, jlong, jlong, jint, jboolean, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_HashSkipListMemTableConfig.h b/java/include/org_forstdb_HashSkipListMemTableConfig.h new file mode 100644 index 000000000..bc800fe5a --- /dev/null +++ b/java/include/org_forstdb_HashSkipListMemTableConfig.h @@ -0,0 +1,27 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_HashSkipListMemTableConfig */ + +#ifndef _Included_org_forstdb_HashSkipListMemTableConfig +#define _Included_org_forstdb_HashSkipListMemTableConfig +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_HashSkipListMemTableConfig_DEFAULT_BUCKET_COUNT +#define org_forstdb_HashSkipListMemTableConfig_DEFAULT_BUCKET_COUNT 1000000L +#undef org_forstdb_HashSkipListMemTableConfig_DEFAULT_BRANCHING_FACTOR +#define org_forstdb_HashSkipListMemTableConfig_DEFAULT_BRANCHING_FACTOR 4L +#undef org_forstdb_HashSkipListMemTableConfig_DEFAULT_HEIGHT +#define org_forstdb_HashSkipListMemTableConfig_DEFAULT_HEIGHT 4L +/* + * Class: org_forstdb_HashSkipListMemTableConfig + * Method: newMemTableFactoryHandle + * Signature: (JII)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_HashSkipListMemTableConfig_newMemTableFactoryHandle + (JNIEnv *, jobject, jlong, jint, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_HyperClockCache.h b/java/include/org_forstdb_HyperClockCache.h new file mode 100644 index 000000000..c7f5ea634 --- /dev/null +++ b/java/include/org_forstdb_HyperClockCache.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_HyperClockCache */ + +#ifndef _Included_org_forstdb_HyperClockCache +#define _Included_org_forstdb_HyperClockCache +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_HyperClockCache + * Method: disposeInternalJni + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_HyperClockCache_disposeInternalJni + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_HyperClockCache + * Method: newHyperClockCache + * Signature: (JJIZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_HyperClockCache_newHyperClockCache + (JNIEnv *, jclass, jlong, jlong, jint, jboolean); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ImportColumnFamilyOptions.h b/java/include/org_forstdb_ImportColumnFamilyOptions.h new file mode 100644 index 000000000..d97b72abb --- /dev/null +++ b/java/include/org_forstdb_ImportColumnFamilyOptions.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ImportColumnFamilyOptions */ + +#ifndef _Included_org_forstdb_ImportColumnFamilyOptions +#define _Included_org_forstdb_ImportColumnFamilyOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ImportColumnFamilyOptions + * Method: newImportColumnFamilyOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ImportColumnFamilyOptions_newImportColumnFamilyOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_ImportColumnFamilyOptions + * Method: moveFiles + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ImportColumnFamilyOptions_moveFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ImportColumnFamilyOptions + * Method: setMoveFiles + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ImportColumnFamilyOptions_setMoveFiles + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ImportColumnFamilyOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ImportColumnFamilyOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_IngestExternalFileOptions.h b/java/include/org_forstdb_IngestExternalFileOptions.h new file mode 100644 index 000000000..7db0ec878 --- /dev/null +++ b/java/include/org_forstdb_IngestExternalFileOptions.h @@ -0,0 +1,133 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_IngestExternalFileOptions */ + +#ifndef _Included_org_forstdb_IngestExternalFileOptions +#define _Included_org_forstdb_IngestExternalFileOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: newIngestExternalFileOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_IngestExternalFileOptions_newIngestExternalFileOptions__ + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: newIngestExternalFileOptions + * Signature: (ZZZZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_IngestExternalFileOptions_newIngestExternalFileOptions__ZZZZ + (JNIEnv *, jclass, jboolean, jboolean, jboolean, jboolean); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_IngestExternalFileOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: moveFiles + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_IngestExternalFileOptions_moveFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: setMoveFiles + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_IngestExternalFileOptions_setMoveFiles + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: snapshotConsistency + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_IngestExternalFileOptions_snapshotConsistency + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: setSnapshotConsistency + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_IngestExternalFileOptions_setSnapshotConsistency + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: allowGlobalSeqNo + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_IngestExternalFileOptions_allowGlobalSeqNo + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: setAllowGlobalSeqNo + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_IngestExternalFileOptions_setAllowGlobalSeqNo + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: allowBlockingFlush + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_IngestExternalFileOptions_allowBlockingFlush + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: setAllowBlockingFlush + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_IngestExternalFileOptions_setAllowBlockingFlush + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: ingestBehind + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_IngestExternalFileOptions_ingestBehind + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: setIngestBehind + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_IngestExternalFileOptions_setIngestBehind + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: writeGlobalSeqno + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_IngestExternalFileOptions_writeGlobalSeqno + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_IngestExternalFileOptions + * Method: setWriteGlobalSeqno + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_IngestExternalFileOptions_setWriteGlobalSeqno + (JNIEnv *, jobject, jlong, jboolean); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_LRUCache.h b/java/include/org_forstdb_LRUCache.h new file mode 100644 index 000000000..168288330 --- /dev/null +++ b/java/include/org_forstdb_LRUCache.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_LRUCache */ + +#ifndef _Included_org_forstdb_LRUCache +#define _Included_org_forstdb_LRUCache +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_LRUCache + * Method: newLRUCache + * Signature: (JIZDD)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_LRUCache_newLRUCache + (JNIEnv *, jclass, jlong, jint, jboolean, jdouble, jdouble); + +/* + * Class: org_forstdb_LRUCache + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_LRUCache_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_LiveFileMetaData.h b/java/include/org_forstdb_LiveFileMetaData.h new file mode 100644 index 000000000..f89568b61 --- /dev/null +++ b/java/include/org_forstdb_LiveFileMetaData.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_LiveFileMetaData */ + +#ifndef _Included_org_forstdb_LiveFileMetaData +#define _Included_org_forstdb_LiveFileMetaData +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_LiveFileMetaData + * Method: newLiveFileMetaDataHandle + * Signature: ([BIILjava/lang/String;Ljava/lang/String;JJJ[BI[BIJZJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_LiveFileMetaData_newLiveFileMetaDataHandle + (JNIEnv *, jobject, jbyteArray, jint, jint, jstring, jstring, jlong, jlong, jlong, jbyteArray, jint, jbyteArray, jint, jlong, jboolean, jlong, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Logger.h b/java/include/org_forstdb_Logger.h new file mode 100644 index 000000000..d1968a3fd --- /dev/null +++ b/java/include/org_forstdb_Logger.h @@ -0,0 +1,57 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Logger */ + +#ifndef _Included_org_forstdb_Logger +#define _Included_org_forstdb_Logger +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_Logger_WITH_OPTIONS +#define org_forstdb_Logger_WITH_OPTIONS 0LL +#undef org_forstdb_Logger_WITH_DBOPTIONS +#define org_forstdb_Logger_WITH_DBOPTIONS 1LL +/* + * Class: org_forstdb_Logger + * Method: createNewLoggerOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Logger_createNewLoggerOptions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Logger + * Method: createNewLoggerDbOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Logger_createNewLoggerDbOptions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Logger + * Method: setInfoLogLevel + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Logger_setInfoLogLevel + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Logger + * Method: infoLogLevel + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Logger_infoLogLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Logger + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Logger_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_MemoryUtil.h b/java/include/org_forstdb_MemoryUtil.h new file mode 100644 index 000000000..ed7b3fd3f --- /dev/null +++ b/java/include/org_forstdb_MemoryUtil.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_MemoryUtil */ + +#ifndef _Included_org_forstdb_MemoryUtil +#define _Included_org_forstdb_MemoryUtil +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_MemoryUtil + * Method: getApproximateMemoryUsageByType + * Signature: ([J[J)Ljava/util/Map; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_MemoryUtil_getApproximateMemoryUsageByType + (JNIEnv *, jclass, jlongArray, jlongArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_NativeComparatorWrapper.h b/java/include/org_forstdb_NativeComparatorWrapper.h new file mode 100644 index 000000000..7fb7fb9d1 --- /dev/null +++ b/java/include/org_forstdb_NativeComparatorWrapper.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_NativeComparatorWrapper */ + +#ifndef _Included_org_forstdb_NativeComparatorWrapper +#define _Included_org_forstdb_NativeComparatorWrapper +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_NativeComparatorWrapper + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_NativeComparatorWrapper_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper.h b/java/include/org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper.h new file mode 100644 index 000000000..b94d5e91a --- /dev/null +++ b/java/include/org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper */ + +#ifndef _Included_org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper +#define _Included_org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper + * Method: newStringComparator + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_NativeComparatorWrapperTest_00024NativeStringComparatorWrapper_newStringComparator + (JNIEnv *, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_OptimisticTransactionDB.h b/java/include/org_forstdb_OptimisticTransactionDB.h new file mode 100644 index 000000000..86f111d7b --- /dev/null +++ b/java/include/org_forstdb_OptimisticTransactionDB.h @@ -0,0 +1,87 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_OptimisticTransactionDB */ + +#ifndef _Included_org_forstdb_OptimisticTransactionDB +#define _Included_org_forstdb_OptimisticTransactionDB +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_OptimisticTransactionDB_NOT_FOUND +#define org_forstdb_OptimisticTransactionDB_NOT_FOUND -1L +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_OptimisticTransactionDB_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: open + * Signature: (JLjava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_OptimisticTransactionDB_open__JLjava_lang_String_2 + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: open + * Signature: (JLjava/lang/String;[[B[J)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_OptimisticTransactionDB_open__JLjava_lang_String_2_3_3B_3J + (JNIEnv *, jclass, jlong, jstring, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: closeDatabase + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_OptimisticTransactionDB_closeDatabase + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: beginTransaction + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_OptimisticTransactionDB_beginTransaction__JJ + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: beginTransaction + * Signature: (JJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_OptimisticTransactionDB_beginTransaction__JJJ + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: beginTransaction_withOld + * Signature: (JJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJ + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: beginTransaction_withOld + * Signature: (JJJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJJ + (JNIEnv *, jobject, jlong, jlong, jlong, jlong); + +/* + * Class: org_forstdb_OptimisticTransactionDB + * Method: getBaseDB + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_OptimisticTransactionDB_getBaseDB + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_OptimisticTransactionOptions.h b/java/include/org_forstdb_OptimisticTransactionOptions.h new file mode 100644 index 000000000..9060f1b13 --- /dev/null +++ b/java/include/org_forstdb_OptimisticTransactionOptions.h @@ -0,0 +1,53 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_OptimisticTransactionOptions */ + +#ifndef _Included_org_forstdb_OptimisticTransactionOptions +#define _Included_org_forstdb_OptimisticTransactionOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_OptimisticTransactionOptions + * Method: newOptimisticTransactionOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_OptimisticTransactionOptions_newOptimisticTransactionOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_OptimisticTransactionOptions + * Method: isSetSnapshot + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_OptimisticTransactionOptions_isSetSnapshot + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_OptimisticTransactionOptions + * Method: setSetSnapshot + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_OptimisticTransactionOptions_setSetSnapshot + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_OptimisticTransactionOptions + * Method: setComparator + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_OptimisticTransactionOptions_setComparator + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_OptimisticTransactionOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_OptimisticTransactionOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Options.h b/java/include/org_forstdb_Options.h new file mode 100644 index 000000000..363a38321 --- /dev/null +++ b/java/include/org_forstdb_Options.h @@ -0,0 +1,2405 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Options */ + +#ifndef _Included_org_forstdb_Options +#define _Included_org_forstdb_Options +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Options + * Method: newOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_newOptions__ + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_Options + * Method: newOptions + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_newOptions__JJ + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: copyOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_copyOptions + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setEnv + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setEnv + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: prepareForBulkLoad + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_prepareForBulkLoad + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setIncreaseParallelism + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setIncreaseParallelism + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: setCreateIfMissing + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCreateIfMissing + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: createIfMissing + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_createIfMissing + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCreateMissingColumnFamilies + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCreateMissingColumnFamilies + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: createMissingColumnFamilies + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_createMissingColumnFamilies + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setErrorIfExists + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setErrorIfExists + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: errorIfExists + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_errorIfExists + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setParanoidChecks + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setParanoidChecks + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: paranoidChecks + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_paranoidChecks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setRateLimiter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setRateLimiter + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setSstFileManager + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setSstFileManager + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLogger + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLogger + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setInfoLogLevel + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setInfoLogLevel + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: infoLogLevel + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_infoLogLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxOpenFiles + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxOpenFiles + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxOpenFiles + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxOpenFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxTotalWalSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxTotalWalSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxFileOpeningThreads + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxFileOpeningThreads + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxFileOpeningThreads + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxFileOpeningThreads + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxTotalWalSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxTotalWalSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setStatistics + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setStatistics + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: statistics + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_statistics + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: useFsync + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_useFsync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setUseFsync + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setUseFsync + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: setDbPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setDbPaths + (JNIEnv *, jobject, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_Options + * Method: dbPathsLen + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_dbPathsLen + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: dbPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_dbPaths + (JNIEnv *, jobject, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_Options + * Method: setDbLogDir + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setDbLogDir + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_Options + * Method: dbLogDir + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_Options_dbLogDir + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWalDir + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWalDir + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_Options + * Method: walDir + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_Options_walDir + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setDeleteObsoleteFilesPeriodMicros + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setDeleteObsoleteFilesPeriodMicros + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: deleteObsoleteFilesPeriodMicros + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_deleteObsoleteFilesPeriodMicros + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxBackgroundCompactions + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxBackgroundCompactions + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxBackgroundCompactions + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxBackgroundCompactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxSubcompactions + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxSubcompactions + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxSubcompactions + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxSubcompactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxBackgroundFlushes + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxBackgroundFlushes + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxBackgroundFlushes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxBackgroundFlushes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxBackgroundJobs + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxBackgroundJobs + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxBackgroundJobs + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxBackgroundJobs + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxLogFileSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxLogFileSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxLogFileSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxLogFileSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLogFileTimeToRoll + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLogFileTimeToRoll + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: logFileTimeToRoll + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_logFileTimeToRoll + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setKeepLogFileNum + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setKeepLogFileNum + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: keepLogFileNum + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_keepLogFileNum + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setRecycleLogFileNum + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setRecycleLogFileNum + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: recycleLogFileNum + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_recycleLogFileNum + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxManifestFileSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxManifestFileSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxManifestFileSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxManifestFileSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxTableFilesSizeFIFO + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxTableFilesSizeFIFO + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxTableFilesSizeFIFO + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxTableFilesSizeFIFO + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setTableCacheNumshardbits + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setTableCacheNumshardbits + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: tableCacheNumshardbits + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_tableCacheNumshardbits + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWalTtlSeconds + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWalTtlSeconds + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: walTtlSeconds + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_walTtlSeconds + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWalSizeLimitMB + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWalSizeLimitMB + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: walSizeLimitMB + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_walSizeLimitMB + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxWriteBatchGroupSizeBytes + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxWriteBatchGroupSizeBytes + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxWriteBatchGroupSizeBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxWriteBatchGroupSizeBytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setManifestPreallocationSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setManifestPreallocationSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: manifestPreallocationSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_manifestPreallocationSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setUseDirectReads + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setUseDirectReads + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: useDirectReads + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_useDirectReads + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setUseDirectIoForFlushAndCompaction + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setUseDirectIoForFlushAndCompaction + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: useDirectIoForFlushAndCompaction + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_useDirectIoForFlushAndCompaction + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAllowFAllocate + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAllowFAllocate + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: allowFAllocate + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_allowFAllocate + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAllowMmapReads + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAllowMmapReads + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: allowMmapReads + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_allowMmapReads + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAllowMmapWrites + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAllowMmapWrites + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: allowMmapWrites + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_allowMmapWrites + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setIsFdCloseOnExec + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setIsFdCloseOnExec + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: isFdCloseOnExec + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_isFdCloseOnExec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setStatsDumpPeriodSec + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setStatsDumpPeriodSec + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: statsDumpPeriodSec + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_statsDumpPeriodSec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setStatsPersistPeriodSec + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setStatsPersistPeriodSec + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: statsPersistPeriodSec + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_statsPersistPeriodSec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setStatsHistoryBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setStatsHistoryBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: statsHistoryBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_statsHistoryBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAdviseRandomOnOpen + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAdviseRandomOnOpen + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: adviseRandomOnOpen + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_adviseRandomOnOpen + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setDbWriteBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setDbWriteBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWriteBufferManager + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWriteBufferManager + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: dbWriteBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_dbWriteBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAccessHintOnCompactionStart + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAccessHintOnCompactionStart + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: accessHintOnCompactionStart + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_accessHintOnCompactionStart + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompactionReadaheadSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompactionReadaheadSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: compactionReadaheadSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_compactionReadaheadSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setRandomAccessMaxBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setRandomAccessMaxBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: randomAccessMaxBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_randomAccessMaxBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWritableFileMaxBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWritableFileMaxBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: writableFileMaxBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_writableFileMaxBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setUseAdaptiveMutex + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setUseAdaptiveMutex + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: useAdaptiveMutex + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_useAdaptiveMutex + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBytesPerSync + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBytesPerSync + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: bytesPerSync + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_bytesPerSync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWalBytesPerSync + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWalBytesPerSync + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: walBytesPerSync + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_walBytesPerSync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setStrictBytesPerSync + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setStrictBytesPerSync + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: strictBytesPerSync + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_strictBytesPerSync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setEventListeners + * Signature: (J[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setEventListeners + (JNIEnv *, jclass, jlong, jlongArray); + +/* + * Class: org_forstdb_Options + * Method: eventListeners + * Signature: (J)[Lorg/forstdb/AbstractEventListener; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_Options_eventListeners + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setEnableThreadTracking + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setEnableThreadTracking + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: enableThreadTracking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_enableThreadTracking + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setDelayedWriteRate + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setDelayedWriteRate + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: delayedWriteRate + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_delayedWriteRate + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setEnablePipelinedWrite + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setEnablePipelinedWrite + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: enablePipelinedWrite + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_enablePipelinedWrite + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setUnorderedWrite + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setUnorderedWrite + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: unorderedWrite + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_unorderedWrite + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAllowConcurrentMemtableWrite + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAllowConcurrentMemtableWrite + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: allowConcurrentMemtableWrite + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_allowConcurrentMemtableWrite + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setEnableWriteThreadAdaptiveYield + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setEnableWriteThreadAdaptiveYield + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: enableWriteThreadAdaptiveYield + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_enableWriteThreadAdaptiveYield + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWriteThreadMaxYieldUsec + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWriteThreadMaxYieldUsec + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: writeThreadMaxYieldUsec + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_writeThreadMaxYieldUsec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWriteThreadSlowYieldUsec + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWriteThreadSlowYieldUsec + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: writeThreadSlowYieldUsec + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_writeThreadSlowYieldUsec + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setSkipStatsUpdateOnDbOpen + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setSkipStatsUpdateOnDbOpen + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: skipStatsUpdateOnDbOpen + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_skipStatsUpdateOnDbOpen + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setSkipCheckingSstFileSizesOnDbOpen + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setSkipCheckingSstFileSizesOnDbOpen + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: skipCheckingSstFileSizesOnDbOpen + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_skipCheckingSstFileSizesOnDbOpen + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWalRecoveryMode + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWalRecoveryMode + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: walRecoveryMode + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_walRecoveryMode + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAllow2pc + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAllow2pc + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: allow2pc + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_allow2pc + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setRowCache + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setRowCache + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWalFilter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWalFilter + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setFailIfOptionsFileError + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setFailIfOptionsFileError + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: failIfOptionsFileError + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_failIfOptionsFileError + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setDumpMallocStats + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setDumpMallocStats + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: dumpMallocStats + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_dumpMallocStats + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAvoidFlushDuringRecovery + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAvoidFlushDuringRecovery + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: avoidFlushDuringRecovery + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_avoidFlushDuringRecovery + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAvoidFlushDuringShutdown + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAvoidFlushDuringShutdown + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: avoidFlushDuringShutdown + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_avoidFlushDuringShutdown + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAllowIngestBehind + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAllowIngestBehind + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: allowIngestBehind + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_allowIngestBehind + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setTwoWriteQueues + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setTwoWriteQueues + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: twoWriteQueues + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_twoWriteQueues + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setManualWalFlush + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setManualWalFlush + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: manualWalFlush + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_manualWalFlush + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: oldDefaults + * Signature: (JII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_oldDefaults + (JNIEnv *, jclass, jlong, jint, jint); + +/* + * Class: org_forstdb_Options + * Method: optimizeForSmallDb + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_optimizeForSmallDb__J + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: optimizeForSmallDb + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_optimizeForSmallDb__JJ + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: optimizeForPointLookup + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_optimizeForPointLookup + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: optimizeLevelStyleCompaction + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_optimizeLevelStyleCompaction + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: optimizeUniversalStyleCompaction + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_optimizeUniversalStyleCompaction + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setComparatorHandle + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setComparatorHandle__JI + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: setComparatorHandle + * Signature: (JJB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setComparatorHandle__JJB + (JNIEnv *, jobject, jlong, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: setMergeOperatorName + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMergeOperatorName + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_Options + * Method: setMergeOperator + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMergeOperator + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompactionFilterHandle + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompactionFilterHandle + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompactionFilterFactoryHandle + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompactionFilterFactoryHandle + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWriteBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWriteBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: writeBufferSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_writeBufferSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxWriteBufferNumber + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxWriteBufferNumber + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxWriteBufferNumber + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxWriteBufferNumber + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMinWriteBufferNumberToMerge + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMinWriteBufferNumberToMerge + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: minWriteBufferNumberToMerge + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_minWriteBufferNumberToMerge + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompressionType + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompressionType + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: compressionType + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_compressionType + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompressionPerLevel + * Signature: (J[B)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompressionPerLevel + (JNIEnv *, jobject, jlong, jbyteArray); + +/* + * Class: org_forstdb_Options + * Method: compressionPerLevel + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_Options_compressionPerLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBottommostCompressionType + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBottommostCompressionType + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: bottommostCompressionType + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_bottommostCompressionType + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBottommostCompressionOptions + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBottommostCompressionOptions + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompressionOptions + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompressionOptions + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: useFixedLengthPrefixExtractor + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_useFixedLengthPrefixExtractor + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: useCappedPrefixExtractor + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_useCappedPrefixExtractor + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: setNumLevels + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setNumLevels + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: numLevels + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_numLevels + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLevelZeroFileNumCompactionTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLevelZeroFileNumCompactionTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: levelZeroFileNumCompactionTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_levelZeroFileNumCompactionTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLevelZeroSlowdownWritesTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLevelZeroSlowdownWritesTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: levelZeroSlowdownWritesTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_levelZeroSlowdownWritesTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLevelZeroStopWritesTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLevelZeroStopWritesTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: levelZeroStopWritesTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_levelZeroStopWritesTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setTargetFileSizeBase + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setTargetFileSizeBase + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: targetFileSizeBase + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_targetFileSizeBase + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setTargetFileSizeMultiplier + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setTargetFileSizeMultiplier + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: targetFileSizeMultiplier + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_targetFileSizeMultiplier + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxBytesForLevelBase + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxBytesForLevelBase + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxBytesForLevelBase + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxBytesForLevelBase + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLevelCompactionDynamicLevelBytes + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLevelCompactionDynamicLevelBytes + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: levelCompactionDynamicLevelBytes + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_levelCompactionDynamicLevelBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxBytesForLevelMultiplier + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxBytesForLevelMultiplier + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_Options + * Method: maxBytesForLevelMultiplier + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_Options_maxBytesForLevelMultiplier + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxCompactionBytes + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxCompactionBytes + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxCompactionBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxCompactionBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setArenaBlockSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setArenaBlockSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: arenaBlockSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_arenaBlockSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setDisableAutoCompactions + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setDisableAutoCompactions + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: disableAutoCompactions + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_disableAutoCompactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompactionStyle + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompactionStyle + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: compactionStyle + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_compactionStyle + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxSequentialSkipInIterations + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxSequentialSkipInIterations + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxSequentialSkipInIterations + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxSequentialSkipInIterations + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMemTableFactory + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMemTableFactory + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: memTableFactoryName + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_Options_memTableFactoryName + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setTableFactory + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setTableFactory + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: tableFactoryName + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_Options_tableFactoryName + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCfPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCfPaths + (JNIEnv *, jclass, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_Options + * Method: cfPathsLen + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_cfPathsLen + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: cfPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_cfPaths + (JNIEnv *, jclass, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_Options + * Method: setInplaceUpdateSupport + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setInplaceUpdateSupport + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: inplaceUpdateSupport + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_inplaceUpdateSupport + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setInplaceUpdateNumLocks + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setInplaceUpdateNumLocks + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: inplaceUpdateNumLocks + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_inplaceUpdateNumLocks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMemtablePrefixBloomSizeRatio + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMemtablePrefixBloomSizeRatio + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_Options + * Method: memtablePrefixBloomSizeRatio + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_Options_memtablePrefixBloomSizeRatio + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setExperimentalMempurgeThreshold + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setExperimentalMempurgeThreshold + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_Options + * Method: experimentalMempurgeThreshold + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_Options_experimentalMempurgeThreshold + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMemtableWholeKeyFiltering + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMemtableWholeKeyFiltering + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: memtableWholeKeyFiltering + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_memtableWholeKeyFiltering + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBloomLocality + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBloomLocality + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: bloomLocality + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_bloomLocality + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxSuccessiveMerges + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxSuccessiveMerges + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: maxSuccessiveMerges + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_maxSuccessiveMerges + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setOptimizeFiltersForHits + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setOptimizeFiltersForHits + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: optimizeFiltersForHits + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_optimizeFiltersForHits + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMemtableHugePageSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMemtableHugePageSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: memtableHugePageSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_memtableHugePageSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setSoftPendingCompactionBytesLimit + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setSoftPendingCompactionBytesLimit + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: softPendingCompactionBytesLimit + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_softPendingCompactionBytesLimit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setHardPendingCompactionBytesLimit + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setHardPendingCompactionBytesLimit + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: hardPendingCompactionBytesLimit + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_hardPendingCompactionBytesLimit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLevel0FileNumCompactionTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLevel0FileNumCompactionTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: level0FileNumCompactionTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_level0FileNumCompactionTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLevel0SlowdownWritesTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLevel0SlowdownWritesTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: level0SlowdownWritesTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_level0SlowdownWritesTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLevel0StopWritesTrigger + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLevel0StopWritesTrigger + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: level0StopWritesTrigger + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_level0StopWritesTrigger + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxBytesForLevelMultiplierAdditional + * Signature: (J[I)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxBytesForLevelMultiplierAdditional + (JNIEnv *, jobject, jlong, jintArray); + +/* + * Class: org_forstdb_Options + * Method: maxBytesForLevelMultiplierAdditional + * Signature: (J)[I + */ +JNIEXPORT jintArray JNICALL Java_org_forstdb_Options_maxBytesForLevelMultiplierAdditional + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setParanoidFileChecks + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setParanoidFileChecks + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: paranoidFileChecks + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_paranoidFileChecks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxWriteBufferNumberToMaintain + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxWriteBufferNumberToMaintain + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxWriteBufferNumberToMaintain + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxWriteBufferNumberToMaintain + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompactionPriority + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompactionPriority + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: compactionPriority + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_compactionPriority + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setReportBgIoStats + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setReportBgIoStats + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: reportBgIoStats + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_reportBgIoStats + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setTtl + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setTtl + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: ttl + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_ttl + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setPeriodicCompactionSeconds + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setPeriodicCompactionSeconds + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: periodicCompactionSeconds + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_periodicCompactionSeconds + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompactionOptionsUniversal + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompactionOptionsUniversal + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompactionOptionsFIFO + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompactionOptionsFIFO + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setForceConsistencyChecks + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setForceConsistencyChecks + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: forceConsistencyChecks + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_forceConsistencyChecks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAtomicFlush + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAtomicFlush + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: atomicFlush + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_atomicFlush + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setSstPartitionerFactory + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setSstPartitionerFactory + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMemtableMaxRangeDeletions + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMemtableMaxRangeDeletions + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: memtableMaxRangeDeletions + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_memtableMaxRangeDeletions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setCompactionThreadLimiter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setCompactionThreadLimiter + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: setAvoidUnnecessaryBlockingIO + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setAvoidUnnecessaryBlockingIO + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: avoidUnnecessaryBlockingIO + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_avoidUnnecessaryBlockingIO + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setPersistStatsToDisk + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setPersistStatsToDisk + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: persistStatsToDisk + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_persistStatsToDisk + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setWriteDbidToManifest + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setWriteDbidToManifest + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: writeDbidToManifest + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_writeDbidToManifest + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setLogReadaheadSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setLogReadaheadSize + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: logReadaheadSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_logReadaheadSize + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBestEffortsRecovery + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBestEffortsRecovery + (JNIEnv *, jclass, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: bestEffortsRecovery + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_bestEffortsRecovery + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMaxBgErrorResumeCount + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMaxBgErrorResumeCount + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: maxBgerrorResumeCount + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_maxBgerrorResumeCount + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBgerrorResumeRetryInterval + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBgerrorResumeRetryInterval + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: bgerrorResumeRetryInterval + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_bgerrorResumeRetryInterval + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Options + * Method: setEnableBlobFiles + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setEnableBlobFiles + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: enableBlobFiles + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_enableBlobFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setMinBlobSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setMinBlobSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: minBlobSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_minBlobSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBlobFileSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBlobFileSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: blobFileSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_blobFileSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBlobCompressionType + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBlobCompressionType + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: blobCompressionType + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_blobCompressionType + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setEnableBlobGarbageCollection + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setEnableBlobGarbageCollection + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_Options + * Method: enableBlobGarbageCollection + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Options_enableBlobGarbageCollection + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBlobGarbageCollectionAgeCutoff + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBlobGarbageCollectionAgeCutoff + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_Options + * Method: blobGarbageCollectionAgeCutoff + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_Options_blobGarbageCollectionAgeCutoff + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBlobGarbageCollectionForceThreshold + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBlobGarbageCollectionForceThreshold + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_Options + * Method: blobGarbageCollectionForceThreshold + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_Options_blobGarbageCollectionForceThreshold + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBlobCompactionReadaheadSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBlobCompactionReadaheadSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Options + * Method: blobCompactionReadaheadSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Options_blobCompactionReadaheadSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setBlobFileStartingLevel + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setBlobFileStartingLevel + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Options + * Method: blobFileStartingLevel + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Options_blobFileStartingLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Options + * Method: setPrepopulateBlobCache + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Options_setPrepopulateBlobCache + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Options + * Method: prepopulateBlobCache + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Options_prepopulateBlobCache + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_OptionsUtil.h b/java/include/org_forstdb_OptionsUtil.h new file mode 100644 index 000000000..e4bb85ab0 --- /dev/null +++ b/java/include/org_forstdb_OptionsUtil.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_OptionsUtil */ + +#ifndef _Included_org_forstdb_OptionsUtil +#define _Included_org_forstdb_OptionsUtil +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_OptionsUtil + * Method: loadLatestOptions + * Signature: (JLjava/lang/String;JLjava/util/List;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_OptionsUtil_loadLatestOptions + (JNIEnv *, jclass, jlong, jstring, jlong, jobject); + +/* + * Class: org_forstdb_OptionsUtil + * Method: loadOptionsFromFile + * Signature: (JLjava/lang/String;JLjava/util/List;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_OptionsUtil_loadOptionsFromFile + (JNIEnv *, jclass, jlong, jstring, jlong, jobject); + +/* + * Class: org_forstdb_OptionsUtil + * Method: getLatestOptionsFileName + * Signature: (Ljava/lang/String;J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_OptionsUtil_getLatestOptionsFileName + (JNIEnv *, jclass, jstring, jlong); + +/* + * Class: org_forstdb_OptionsUtil + * Method: readTableFormatConfig + * Signature: (J)Lorg/forstdb/TableFormatConfig; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_OptionsUtil_readTableFormatConfig + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_PerfContext.h b/java/include/org_forstdb_PerfContext.h new file mode 100644 index 000000000..50f9155a9 --- /dev/null +++ b/java/include/org_forstdb_PerfContext.h @@ -0,0 +1,805 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_PerfContext */ + +#ifndef _Included_org_forstdb_PerfContext +#define _Included_org_forstdb_PerfContext +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_PerfContext + * Method: reset + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_PerfContext_reset + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getUserKeyComparisonCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getUserKeyComparisonCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheHitCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockCacheHitCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockReadCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockReadCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockReadByte + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockReadByte + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockReadTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockReadTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockReadCpuTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockReadCpuTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheIndexHitCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockCacheIndexHitCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheStandaloneHandleCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockCacheStandaloneHandleCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheRealHandleCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockCacheRealHandleCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getIndexBlockReadCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getIndexBlockReadCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockCacheFilterHitCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockCacheFilterHitCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getFilterBlockReadCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getFilterBlockReadCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressionDictBlockReadCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getCompressionDictBlockReadCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSecondaryCacheHitCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSecondaryCacheHitCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressedSecCacheInsertRealCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getCompressedSecCacheInsertRealCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressedSecCacheInsertDummyCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getCompressedSecCacheInsertDummyCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressedSecCacheUncompressedBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getCompressedSecCacheUncompressedBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getCompressedSecCacheCompressedBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getCompressedSecCacheCompressedBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockChecksumTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockChecksumTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockDecompressTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockDecompressTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getReadBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getReadBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getMultigetReadBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getMultigetReadBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getIterReadBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getIterReadBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobCacheHitCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlobCacheHitCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobReadCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlobReadCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobReadByte + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlobReadByte + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobReadTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlobReadTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobChecksumTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlobChecksumTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlobDecompressTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlobDecompressTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalKeySkippedCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getInternalKeySkippedCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalDeleteSkippedCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getInternalDeleteSkippedCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalRecentSkippedCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getInternalRecentSkippedCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalMergeCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getInternalMergeCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalMergePointLookupCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getInternalMergePointLookupCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getInternalRangeDelReseekCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getInternalRangeDelReseekCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSnapshotTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSnapshotTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getFromMemtableTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getFromMemtableTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getFromMemtableCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getFromMemtableCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getPostProcessTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getPostProcessTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getFromOutputFilesTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getFromOutputFilesTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekOnMemtableTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSeekOnMemtableTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekOnMemtableCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSeekOnMemtableCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getNextOnMemtableCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getNextOnMemtableCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getPrevOnMemtableCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getPrevOnMemtableCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekChildSeekTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSeekChildSeekTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekChildSeekCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSeekChildSeekCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekMinHeapTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSeekMinHeapTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekMaxHeapTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSeekMaxHeapTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getSeekInternalSeekTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getSeekInternalSeekTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getFindNextUserEntryTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getFindNextUserEntryTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteWalTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getWriteWalTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteMemtableTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getWriteMemtableTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteDelayTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getWriteDelayTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteSchedulingFlushesCompactionsTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getWriteSchedulingFlushesCompactionsTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getWritePreAndPostProcessTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getWritePreAndPostProcessTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getWriteThreadWaitNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getWriteThreadWaitNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getDbMutexLockNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getDbMutexLockNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getDbConditionWaitNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getDbConditionWaitNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getMergeOperatorTimeNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getMergeOperatorTimeNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getReadIndexBlockNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getReadIndexBlockNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getReadFilterBlockNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getReadFilterBlockNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getNewTableBlockIterNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getNewTableBlockIterNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getNewTableIteratorNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getNewTableIteratorNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBlockSeekNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBlockSeekNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getFindTableNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getFindTableNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBloomMemtableHitCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBloomMemtableHitCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBloomMemtableMissCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBloomMemtableMissCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBloomSstHitCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBloomSstHitCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getBloomSstMissCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getBloomSstMissCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getKeyLockWaitTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getKeyLockWaitTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getKeyLockWaitCount + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getKeyLockWaitCount + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewSequentialFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvNewSequentialFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewRandomAccessFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvNewRandomAccessFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewWritableFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvNewWritableFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvReuseWritableFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvReuseWritableFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewRandomRwFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvNewRandomRwFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewDirectoryNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvNewDirectoryNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvFileExistsNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvFileExistsNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvGetChildrenNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvGetChildrenNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvGetChildrenFileAttributesNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvGetChildrenFileAttributesNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvDeleteFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvDeleteFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvCreateDirNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvCreateDirNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvCreateDirIfMissingNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvCreateDirIfMissingNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvDeleteDirNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvDeleteDirNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvGetFileSizeNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvGetFileSizeNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvGetFileModificationTimeNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvGetFileModificationTimeNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvRenameFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvRenameFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvLinkFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvLinkFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvLockFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvLockFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvUnlockFileNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvUnlockFileNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEnvNewLoggerNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEnvNewLoggerNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getGetCpuNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getGetCpuNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getIterNextCpuNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getIterNextCpuNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getIterPrevCpuNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getIterPrevCpuNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getIterSeekCpuNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getIterSeekCpuNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getEncryptDataNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getEncryptDataNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getDecryptDataNanos + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getDecryptDataNanos + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_PerfContext + * Method: getNumberAsyncSeek + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PerfContext_getNumberAsyncSeek + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_PersistentCache.h b/java/include/org_forstdb_PersistentCache.h new file mode 100644 index 000000000..a0358f656 --- /dev/null +++ b/java/include/org_forstdb_PersistentCache.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_PersistentCache */ + +#ifndef _Included_org_forstdb_PersistentCache +#define _Included_org_forstdb_PersistentCache +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_PersistentCache + * Method: newPersistentCache + * Signature: (JLjava/lang/String;JJZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PersistentCache_newPersistentCache + (JNIEnv *, jclass, jlong, jstring, jlong, jlong, jboolean); + +/* + * Class: org_forstdb_PersistentCache + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_PersistentCache_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_PlainTableConfig.h b/java/include/org_forstdb_PlainTableConfig.h new file mode 100644 index 000000000..5be3e76aa --- /dev/null +++ b/java/include/org_forstdb_PlainTableConfig.h @@ -0,0 +1,35 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_PlainTableConfig */ + +#ifndef _Included_org_forstdb_PlainTableConfig +#define _Included_org_forstdb_PlainTableConfig +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_PlainTableConfig_VARIABLE_LENGTH +#define org_forstdb_PlainTableConfig_VARIABLE_LENGTH 0L +#undef org_forstdb_PlainTableConfig_DEFAULT_BLOOM_BITS_PER_KEY +#define org_forstdb_PlainTableConfig_DEFAULT_BLOOM_BITS_PER_KEY 10L +#undef org_forstdb_PlainTableConfig_DEFAULT_HASH_TABLE_RATIO +#define org_forstdb_PlainTableConfig_DEFAULT_HASH_TABLE_RATIO 0.75 +#undef org_forstdb_PlainTableConfig_DEFAULT_INDEX_SPARSENESS +#define org_forstdb_PlainTableConfig_DEFAULT_INDEX_SPARSENESS 16L +#undef org_forstdb_PlainTableConfig_DEFAULT_HUGE_TLB_SIZE +#define org_forstdb_PlainTableConfig_DEFAULT_HUGE_TLB_SIZE 0L +#undef org_forstdb_PlainTableConfig_DEFAULT_FULL_SCAN_MODE +#define org_forstdb_PlainTableConfig_DEFAULT_FULL_SCAN_MODE 0L +#undef org_forstdb_PlainTableConfig_DEFAULT_STORE_INDEX_IN_FILE +#define org_forstdb_PlainTableConfig_DEFAULT_STORE_INDEX_IN_FILE 0L +/* + * Class: org_forstdb_PlainTableConfig + * Method: newTableFactoryHandle + * Signature: (IIDIIBZZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_PlainTableConfig_newTableFactoryHandle + (JNIEnv *, jobject, jint, jint, jdouble, jint, jint, jbyte, jboolean, jboolean); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RateLimiter.h b/java/include/org_forstdb_RateLimiter.h new file mode 100644 index 000000000..8cdab2a11 --- /dev/null +++ b/java/include/org_forstdb_RateLimiter.h @@ -0,0 +1,83 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RateLimiter */ + +#ifndef _Included_org_forstdb_RateLimiter +#define _Included_org_forstdb_RateLimiter +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_RateLimiter_DEFAULT_REFILL_PERIOD_MICROS +#define org_forstdb_RateLimiter_DEFAULT_REFILL_PERIOD_MICROS 100000LL +#undef org_forstdb_RateLimiter_DEFAULT_FAIRNESS +#define org_forstdb_RateLimiter_DEFAULT_FAIRNESS 10L +#undef org_forstdb_RateLimiter_DEFAULT_AUTOTUNE +#define org_forstdb_RateLimiter_DEFAULT_AUTOTUNE 0L +/* + * Class: org_forstdb_RateLimiter + * Method: newRateLimiterHandle + * Signature: (JJIBZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RateLimiter_newRateLimiterHandle + (JNIEnv *, jclass, jlong, jlong, jint, jbyte, jboolean); + +/* + * Class: org_forstdb_RateLimiter + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RateLimiter_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RateLimiter + * Method: setBytesPerSecond + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RateLimiter_setBytesPerSecond + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RateLimiter + * Method: getBytesPerSecond + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RateLimiter_getBytesPerSecond + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RateLimiter + * Method: request + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RateLimiter_request + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RateLimiter + * Method: getSingleBurstBytes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RateLimiter_getSingleBurstBytes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RateLimiter + * Method: getTotalBytesThrough + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RateLimiter_getTotalBytesThrough + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RateLimiter + * Method: getTotalRequests + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RateLimiter_getTotalRequests + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ReadOptions.h b/java/include/org_forstdb_ReadOptions.h new file mode 100644 index 000000000..7082dc8c1 --- /dev/null +++ b/java/include/org_forstdb_ReadOptions.h @@ -0,0 +1,389 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ReadOptions */ + +#ifndef _Included_org_forstdb_ReadOptions +#define _Included_org_forstdb_ReadOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ReadOptions + * Method: newReadOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_newReadOptions__ + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_ReadOptions + * Method: newReadOptions + * Signature: (ZZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_newReadOptions__ZZ + (JNIEnv *, jclass, jboolean, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: copyReadOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_copyReadOptions + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: verifyChecksums + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_verifyChecksums + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setVerifyChecksums + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setVerifyChecksums + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: fillCache + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_fillCache + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setFillCache + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setFillCache + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: snapshot + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_snapshot + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setSnapshot + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setSnapshot + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: readTier + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_ReadOptions_readTier + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setReadTier + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setReadTier + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_ReadOptions + * Method: tailing + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_tailing + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setTailing + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setTailing + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: managed + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_managed + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setManaged + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setManaged + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: totalOrderSeek + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_totalOrderSeek + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setTotalOrderSeek + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setTotalOrderSeek + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: prefixSameAsStart + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_prefixSameAsStart + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setPrefixSameAsStart + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setPrefixSameAsStart + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: pinData + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_pinData + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setPinData + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setPinData + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: backgroundPurgeOnIteratorCleanup + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_backgroundPurgeOnIteratorCleanup + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setBackgroundPurgeOnIteratorCleanup + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setBackgroundPurgeOnIteratorCleanup + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: readaheadSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_readaheadSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setReadaheadSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setReadaheadSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: maxSkippableInternalKeys + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_maxSkippableInternalKeys + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setMaxSkippableInternalKeys + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setMaxSkippableInternalKeys + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: ignoreRangeDeletions + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_ignoreRangeDeletions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setIgnoreRangeDeletions + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setIgnoreRangeDeletions + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: setIterateUpperBound + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setIterateUpperBound + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: iterateUpperBound + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_iterateUpperBound + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setIterateLowerBound + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setIterateLowerBound + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: iterateLowerBound + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_iterateLowerBound + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setTableFilter + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setTableFilter + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: autoPrefixMode + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_ReadOptions_autoPrefixMode + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setAutoPrefixMode + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setAutoPrefixMode + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_ReadOptions + * Method: timestamp + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_timestamp + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setTimestamp + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setTimestamp + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: iterStartTs + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_iterStartTs + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setIterStartTs + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setIterStartTs + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: deadline + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_deadline + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setDeadline + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setDeadline + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: ioTimeout + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_ioTimeout + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setIoTimeout + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setIoTimeout + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: valueSizeSoftLimit + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_ReadOptions_valueSizeSoftLimit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_ReadOptions + * Method: setValueSizeSoftLimit + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_ReadOptions_setValueSizeSoftLimit + (JNIEnv *, jobject, jlong, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RemoveEmptyValueCompactionFilter.h b/java/include/org_forstdb_RemoveEmptyValueCompactionFilter.h new file mode 100644 index 000000000..0fdf0786d --- /dev/null +++ b/java/include/org_forstdb_RemoveEmptyValueCompactionFilter.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RemoveEmptyValueCompactionFilter */ + +#ifndef _Included_org_forstdb_RemoveEmptyValueCompactionFilter +#define _Included_org_forstdb_RemoveEmptyValueCompactionFilter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_RemoveEmptyValueCompactionFilter + * Method: createNewRemoveEmptyValueCompactionFilter0 + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RemoveEmptyValueCompactionFilter_createNewRemoveEmptyValueCompactionFilter0 + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RestoreOptions.h b/java/include/org_forstdb_RestoreOptions.h new file mode 100644 index 000000000..cb0cfaa96 --- /dev/null +++ b/java/include/org_forstdb_RestoreOptions.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RestoreOptions */ + +#ifndef _Included_org_forstdb_RestoreOptions +#define _Included_org_forstdb_RestoreOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_RestoreOptions + * Method: newRestoreOptions + * Signature: (Z)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RestoreOptions_newRestoreOptions + (JNIEnv *, jclass, jboolean); + +/* + * Class: org_forstdb_RestoreOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RestoreOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RocksCallbackObject.h b/java/include/org_forstdb_RocksCallbackObject.h new file mode 100644 index 000000000..edd63d253 --- /dev/null +++ b/java/include/org_forstdb_RocksCallbackObject.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RocksCallbackObject */ + +#ifndef _Included_org_forstdb_RocksCallbackObject +#define _Included_org_forstdb_RocksCallbackObject +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_RocksCallbackObject + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksCallbackObject_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RocksDB.h b/java/include/org_forstdb_RocksDB.h new file mode 100644 index 000000000..43248af59 --- /dev/null +++ b/java/include/org_forstdb_RocksDB.h @@ -0,0 +1,935 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RocksDB */ + +#ifndef _Included_org_forstdb_RocksDB +#define _Included_org_forstdb_RocksDB +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_RocksDB_NOT_FOUND +#define org_forstdb_RocksDB_NOT_FOUND -1L +/* + * Class: org_forstdb_RocksDB + * Method: open + * Signature: (JLjava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_open__JLjava_lang_String_2 + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: org_forstdb_RocksDB + * Method: open + * Signature: (JLjava/lang/String;[[B[J)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_open__JLjava_lang_String_2_3_3B_3J + (JNIEnv *, jclass, jlong, jstring, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: openROnly + * Signature: (JLjava/lang/String;Z)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_openROnly__JLjava_lang_String_2Z + (JNIEnv *, jclass, jlong, jstring, jboolean); + +/* + * Class: org_forstdb_RocksDB + * Method: openROnly + * Signature: (JLjava/lang/String;[[B[JZ)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3JZ + (JNIEnv *, jclass, jlong, jstring, jobjectArray, jlongArray, jboolean); + +/* + * Class: org_forstdb_RocksDB + * Method: openAsSecondary + * Signature: (JLjava/lang/String;Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_2 + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: org_forstdb_RocksDB + * Method: openAsSecondary + * Signature: (JLjava/lang/String;Ljava/lang/String;[[B[J)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_2_3_3B_3J + (JNIEnv *, jclass, jlong, jstring, jstring, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: closeDatabase + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_closeDatabase + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: listColumnFamilies + * Signature: (JLjava/lang/String;)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_listColumnFamilies + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: org_forstdb_RocksDB + * Method: createColumnFamily + * Signature: (J[BIJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_createColumnFamily + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: createColumnFamilies + * Signature: (JJ[[B)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_createColumnFamilies__JJ_3_3B + (JNIEnv *, jobject, jlong, jlong, jobjectArray); + +/* + * Class: org_forstdb_RocksDB + * Method: createColumnFamilies + * Signature: (J[J[[B)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_createColumnFamilies__J_3J_3_3B + (JNIEnv *, jobject, jlong, jlongArray, jobjectArray); + +/* + * Class: org_forstdb_RocksDB + * Method: createColumnFamilyWithImport + * Signature: (J[BIJJ[J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_createColumnFamilyWithImport + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong, jlong, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: dropColumnFamily + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_dropColumnFamily + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: dropColumnFamilies + * Signature: (J[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_dropColumnFamilies + (JNIEnv *, jobject, jlong, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: put + * Signature: (J[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_put__J_3BII_3BII + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: put + * Signature: (J[BII[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_put__J_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: put + * Signature: (JJ[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_put__JJ_3BII_3BII + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: put + * Signature: (JJ[BII[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_put__JJ_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: delete + * Signature: (J[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_delete__J_3BII + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: delete + * Signature: (J[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_delete__J_3BIIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: delete + * Signature: (JJ[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_delete__JJ_3BII + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: delete + * Signature: (JJ[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_delete__JJ_3BIIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: singleDelete + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_singleDelete__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: singleDelete + * Signature: (J[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_singleDelete__J_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: singleDelete + * Signature: (JJ[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_singleDelete__JJ_3BI + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: singleDelete + * Signature: (JJ[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_singleDelete__JJ_3BIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: deleteRange + * Signature: (J[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_deleteRange__J_3BII_3BII + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: deleteRange + * Signature: (J[BII[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_deleteRange__J_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: deleteRange + * Signature: (JJ[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_deleteRange__JJ_3BII_3BII + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: deleteRange + * Signature: (JJ[BII[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_deleteRange__JJ_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: clipColumnFamily + * Signature: (JJ[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_clipColumnFamily + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: merge + * Signature: (J[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_merge__J_3BII_3BII + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: merge + * Signature: (J[BII[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_merge__J_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: merge + * Signature: (JJ[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_merge__JJ_3BII_3BII + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: merge + * Signature: (JJ[BII[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_merge__JJ_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: mergeDirect + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_mergeDirect + (JNIEnv *, jobject, jlong, jlong, jobject, jint, jint, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: write0 + * Signature: (JJJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_write0 + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: write1 + * Signature: (JJJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_write1 + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: get + * Signature: (J[BII[BII)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_get__J_3BII_3BII + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: get + * Signature: (J[BII[BIIJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_get__J_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: get + * Signature: (JJ[BII[BII)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_get__JJ_3BII_3BII + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: get + * Signature: (JJ[BII[BIIJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_get__JJ_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: get + * Signature: (J[BII)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_RocksDB_get__J_3BII + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: get + * Signature: (J[BIIJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_RocksDB_get__J_3BIIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: get + * Signature: (JJ[BII)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_RocksDB_get__JJ_3BII + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: get + * Signature: (JJ[BIIJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_RocksDB_get__JJ_3BIIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: multiGet + * Signature: (J[[B[I[I)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_multiGet__J_3_3B_3I_3I + (JNIEnv *, jobject, jlong, jobjectArray, jintArray, jintArray); + +/* + * Class: org_forstdb_RocksDB + * Method: multiGet + * Signature: (J[[B[I[I[J)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_multiGet__J_3_3B_3I_3I_3J + (JNIEnv *, jobject, jlong, jobjectArray, jintArray, jintArray, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: multiGet + * Signature: (JJ[[B[I[I)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_multiGet__JJ_3_3B_3I_3I + (JNIEnv *, jobject, jlong, jlong, jobjectArray, jintArray, jintArray); + +/* + * Class: org_forstdb_RocksDB + * Method: multiGet + * Signature: (JJ[[B[I[I[J)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_multiGet__JJ_3_3B_3I_3I_3J + (JNIEnv *, jobject, jlong, jlong, jobjectArray, jintArray, jintArray, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: multiGet + * Signature: (JJ[J[Ljava/nio/ByteBuffer;[I[I[Ljava/nio/ByteBuffer;[I[Lorg/forstdb/Status;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_multiGet__JJ_3J_3Ljava_nio_ByteBuffer_2_3I_3I_3Ljava_nio_ByteBuffer_2_3I_3Lorg_forstdb_Status_2 + (JNIEnv *, jobject, jlong, jlong, jlongArray, jobjectArray, jintArray, jintArray, jobjectArray, jintArray, jobjectArray); + +/* + * Class: org_forstdb_RocksDB + * Method: keyExists + * Signature: (JJJ[BII)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_RocksDB_keyExists + (JNIEnv *, jobject, jlong, jlong, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: keyExistsDirect + * Signature: (JJJLjava/nio/ByteBuffer;II)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_RocksDB_keyExistsDirect + (JNIEnv *, jobject, jlong, jlong, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: keyMayExist + * Signature: (JJJ[BII)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_RocksDB_keyMayExist + (JNIEnv *, jobject, jlong, jlong, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: keyMayExistFoundValue + * Signature: (JJJ[BII)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_keyMayExistFoundValue + (JNIEnv *, jobject, jlong, jlong, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: putDirect + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_putDirect + (JNIEnv *, jobject, jlong, jlong, jobject, jint, jint, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: iterator + * Signature: (JJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_iterator + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: iterators + * Signature: (J[JJ)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_iterators + (JNIEnv *, jobject, jlong, jlongArray, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getSnapshot + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_getSnapshot + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: releaseSnapshot + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_releaseSnapshot + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getProperty + * Signature: (JJLjava/lang/String;I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_RocksDB_getProperty + (JNIEnv *, jobject, jlong, jlong, jstring, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: getMapProperty + * Signature: (JJLjava/lang/String;I)Ljava/util/Map; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_RocksDB_getMapProperty + (JNIEnv *, jobject, jlong, jlong, jstring, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: getDirect + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_getDirect + (JNIEnv *, jobject, jlong, jlong, jobject, jint, jint, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: keyMayExistDirect + * Signature: (JJJLjava/nio/ByteBuffer;II)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_RocksDB_keyMayExistDirect + (JNIEnv *, jobject, jlong, jlong, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: keyMayExistDirectFoundValue + * Signature: (JJJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)[I + */ +JNIEXPORT jintArray JNICALL Java_org_forstdb_RocksDB_keyMayExistDirectFoundValue + (JNIEnv *, jobject, jlong, jlong, jlong, jobject, jint, jint, jobject, jint, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: deleteDirect + * Signature: (JJLjava/nio/ByteBuffer;IIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_deleteDirect + (JNIEnv *, jobject, jlong, jlong, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getLongProperty + * Signature: (JJLjava/lang/String;I)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_getLongProperty + (JNIEnv *, jobject, jlong, jlong, jstring, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: resetStats + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_resetStats + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getAggregatedLongProperty + * Signature: (JLjava/lang/String;I)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_getAggregatedLongProperty + (JNIEnv *, jobject, jlong, jstring, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: getApproximateSizes + * Signature: (JJ[JB)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_getApproximateSizes + (JNIEnv *, jobject, jlong, jlong, jlongArray, jbyte); + +/* + * Class: org_forstdb_RocksDB + * Method: getApproximateMemTableStats + * Signature: (JJJJ)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_getApproximateMemTableStats + (JNIEnv *, jobject, jlong, jlong, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: compactRange + * Signature: (J[BI[BIJJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_compactRange + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: setOptions + * Signature: (JJ[Ljava/lang/String;[Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_setOptions + (JNIEnv *, jobject, jlong, jlong, jobjectArray, jobjectArray); + +/* + * Class: org_forstdb_RocksDB + * Method: getOptions + * Signature: (JJ)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_RocksDB_getOptions + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: setDBOptions + * Signature: (J[Ljava/lang/String;[Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_setDBOptions + (JNIEnv *, jobject, jlong, jobjectArray, jobjectArray); + +/* + * Class: org_forstdb_RocksDB + * Method: getDBOptions + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_RocksDB_getDBOptions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: setPerfLevel + * Signature: (B)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_setPerfLevel + (JNIEnv *, jobject, jbyte); + +/* + * Class: org_forstdb_RocksDB + * Method: getPerfLevelNative + * Signature: ()B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_RocksDB_getPerfLevelNative + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_RocksDB + * Method: getPerfContextNative + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_getPerfContextNative + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_RocksDB + * Method: compactFiles + * Signature: (JJJ[Ljava/lang/String;IIJ)[Ljava/lang/String; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_compactFiles + (JNIEnv *, jobject, jlong, jlong, jlong, jobjectArray, jint, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: cancelAllBackgroundWork + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_cancelAllBackgroundWork + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_RocksDB + * Method: pauseBackgroundWork + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_pauseBackgroundWork + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: continueBackgroundWork + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_continueBackgroundWork + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: enableAutoCompaction + * Signature: (J[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_enableAutoCompaction + (JNIEnv *, jobject, jlong, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: numberLevels + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_numberLevels + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: maxMemCompactionLevel + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_maxMemCompactionLevel + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: level0StopWriteTrigger + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_level0StopWriteTrigger + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getName + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_RocksDB_getName + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getEnv + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_getEnv + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: flush + * Signature: (JJ[J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_flush + (JNIEnv *, jobject, jlong, jlong, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: flushWal + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_flushWal + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_RocksDB + * Method: syncWal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_syncWal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getLatestSequenceNumber + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_getLatestSequenceNumber + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: disableFileDeletions + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_disableFileDeletions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: enableFileDeletions + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_enableFileDeletions + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_RocksDB + * Method: getLiveFiles + * Signature: (JZ)[Ljava/lang/String; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_getLiveFiles + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_RocksDB + * Method: getSortedWalFiles + * Signature: (J)[Lorg/forstdb/LogFile; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_getSortedWalFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getUpdatesSince + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_getUpdatesSince + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: deleteFile + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_deleteFile + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_RocksDB + * Method: getLiveFilesMetaData + * Signature: (J)[Lorg/forstdb/LiveFileMetaData; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_RocksDB_getLiveFilesMetaData + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getColumnFamilyMetaData + * Signature: (JJ)Lorg/forstdb/ColumnFamilyMetaData; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_RocksDB_getColumnFamilyMetaData + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: ingestExternalFile + * Signature: (JJ[Ljava/lang/String;IJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_ingestExternalFile + (JNIEnv *, jobject, jlong, jlong, jobjectArray, jint, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: verifyChecksum + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_verifyChecksum + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getDefaultColumnFamily + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksDB_getDefaultColumnFamily + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getPropertiesOfAllTables + * Signature: (JJ)Ljava/util/Map; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_RocksDB_getPropertiesOfAllTables + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: getPropertiesOfTablesInRange + * Signature: (JJ[J)Ljava/util/Map; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_RocksDB_getPropertiesOfTablesInRange + (JNIEnv *, jobject, jlong, jlong, jlongArray); + +/* + * Class: org_forstdb_RocksDB + * Method: suggestCompactRange + * Signature: (JJ)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_RocksDB_suggestCompactRange + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: promoteL0 + * Signature: (JJI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_promoteL0 + (JNIEnv *, jobject, jlong, jlong, jint); + +/* + * Class: org_forstdb_RocksDB + * Method: startTrace + * Signature: (JJJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_startTrace + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: endTrace + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_endTrace + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: tryCatchUpWithPrimary + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_tryCatchUpWithPrimary + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: deleteFilesInRanges + * Signature: (JJ[[BZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_deleteFilesInRanges + (JNIEnv *, jobject, jlong, jlong, jobjectArray, jboolean); + +/* + * Class: org_forstdb_RocksDB + * Method: destroyDB + * Signature: (Ljava/lang/String;J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDB_destroyDB + (JNIEnv *, jclass, jstring, jlong); + +/* + * Class: org_forstdb_RocksDB + * Method: version + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksDB_version + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RocksDBExceptionTest.h b/java/include/org_forstdb_RocksDBExceptionTest.h new file mode 100644 index 000000000..0b707eff1 --- /dev/null +++ b/java/include/org_forstdb_RocksDBExceptionTest.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RocksDBExceptionTest */ + +#ifndef _Included_org_forstdb_RocksDBExceptionTest +#define _Included_org_forstdb_RocksDBExceptionTest +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_RocksDBExceptionTest + * Method: raiseException + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDBExceptionTest_raiseException + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_RocksDBExceptionTest + * Method: raiseExceptionWithStatusCode + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDBExceptionTest_raiseExceptionWithStatusCode + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_RocksDBExceptionTest + * Method: raiseExceptionNoMsgWithStatusCode + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCode + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_RocksDBExceptionTest + * Method: raiseExceptionWithStatusCodeSubCode + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeSubCode + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_RocksDBExceptionTest + * Method: raiseExceptionNoMsgWithStatusCodeSubCode + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCodeSubCode + (JNIEnv *, jobject); + +/* + * Class: org_forstdb_RocksDBExceptionTest + * Method: raiseExceptionWithStatusCodeState + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeState + (JNIEnv *, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RocksEnv.h b/java/include/org_forstdb_RocksEnv.h new file mode 100644 index 000000000..6c9bc74c3 --- /dev/null +++ b/java/include/org_forstdb_RocksEnv.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RocksEnv */ + +#ifndef _Included_org_forstdb_RocksEnv +#define _Included_org_forstdb_RocksEnv +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_RocksEnv + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksEnv_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RocksIterator.h b/java/include/org_forstdb_RocksIterator.h new file mode 100644 index 000000000..f89e51591 --- /dev/null +++ b/java/include/org_forstdb_RocksIterator.h @@ -0,0 +1,173 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RocksIterator */ + +#ifndef _Included_org_forstdb_RocksIterator +#define _Included_org_forstdb_RocksIterator +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_RocksIterator + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: isValid0 + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_RocksIterator_isValid0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: seekToFirst0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_seekToFirst0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: seekToLast0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_seekToLast0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: next0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_next0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: prev0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_prev0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: refresh0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_refresh0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: seek0 + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_seek0 + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: seekForPrev0 + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_seekForPrev0 + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: seekDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_seekDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: seekByteArray0 + * Signature: (J[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_seekByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: seekForPrevDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_seekForPrevDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: seekForPrevByteArray0 + * Signature: (J[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_seekForPrevByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: status0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksIterator_status0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: key0 + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_RocksIterator_key0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: value0 + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_RocksIterator_value0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_RocksIterator + * Method: keyDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksIterator_keyDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: keyByteArray0 + * Signature: (J[BII)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksIterator_keyByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: valueDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksIterator_valueDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_RocksIterator + * Method: valueByteArray0 + * Signature: (J[BII)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_RocksIterator_valueByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_RocksMemEnv.h b/java/include/org_forstdb_RocksMemEnv.h new file mode 100644 index 000000000..b4a080847 --- /dev/null +++ b/java/include/org_forstdb_RocksMemEnv.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_RocksMemEnv */ + +#ifndef _Included_org_forstdb_RocksMemEnv +#define _Included_org_forstdb_RocksMemEnv +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_RocksMemEnv + * Method: createMemEnv + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_RocksMemEnv_createMemEnv + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_RocksMemEnv + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_RocksMemEnv_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_SkipListMemTableConfig.h b/java/include/org_forstdb_SkipListMemTableConfig.h new file mode 100644 index 000000000..43a6f1946 --- /dev/null +++ b/java/include/org_forstdb_SkipListMemTableConfig.h @@ -0,0 +1,23 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_SkipListMemTableConfig */ + +#ifndef _Included_org_forstdb_SkipListMemTableConfig +#define _Included_org_forstdb_SkipListMemTableConfig +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_SkipListMemTableConfig_DEFAULT_LOOKAHEAD +#define org_forstdb_SkipListMemTableConfig_DEFAULT_LOOKAHEAD 0LL +/* + * Class: org_forstdb_SkipListMemTableConfig + * Method: newMemTableFactoryHandle0 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SkipListMemTableConfig_newMemTableFactoryHandle0 + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Slice.h b/java/include/org_forstdb_Slice.h new file mode 100644 index 000000000..45fae672a --- /dev/null +++ b/java/include/org_forstdb_Slice.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Slice */ + +#ifndef _Included_org_forstdb_Slice +#define _Included_org_forstdb_Slice +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Slice + * Method: data0 + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_Slice_data0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Slice + * Method: createNewSlice0 + * Signature: ([BI)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Slice_createNewSlice0 + (JNIEnv *, jclass, jbyteArray, jint); + +/* + * Class: org_forstdb_Slice + * Method: createNewSlice1 + * Signature: ([B)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Slice_createNewSlice1 + (JNIEnv *, jclass, jbyteArray); + +/* + * Class: org_forstdb_Slice + * Method: clear0 + * Signature: (JZJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Slice_clear0 + (JNIEnv *, jobject, jlong, jboolean, jlong); + +/* + * Class: org_forstdb_Slice + * Method: removePrefix0 + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Slice_removePrefix0 + (JNIEnv *, jobject, jlong, jint); + +/* + * Class: org_forstdb_Slice + * Method: disposeInternalBuf + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Slice_disposeInternalBuf + (JNIEnv *, jobject, jlong, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Snapshot.h b/java/include/org_forstdb_Snapshot.h new file mode 100644 index 000000000..595a18e68 --- /dev/null +++ b/java/include/org_forstdb_Snapshot.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Snapshot */ + +#ifndef _Included_org_forstdb_Snapshot +#define _Included_org_forstdb_Snapshot +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Snapshot + * Method: getSequenceNumber + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Snapshot_getSequenceNumber + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_SstFileManager.h b/java/include/org_forstdb_SstFileManager.h new file mode 100644 index 000000000..25fe9e0db --- /dev/null +++ b/java/include/org_forstdb_SstFileManager.h @@ -0,0 +1,117 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_SstFileManager */ + +#ifndef _Included_org_forstdb_SstFileManager +#define _Included_org_forstdb_SstFileManager +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_SstFileManager_RATE_BYTES_PER_SEC_DEFAULT +#define org_forstdb_SstFileManager_RATE_BYTES_PER_SEC_DEFAULT 0LL +#undef org_forstdb_SstFileManager_DELETE_EXISTING_TRASH_DEFAULT +#define org_forstdb_SstFileManager_DELETE_EXISTING_TRASH_DEFAULT 1L +#undef org_forstdb_SstFileManager_MAX_TRASH_DB_RATION_DEFAULT +#define org_forstdb_SstFileManager_MAX_TRASH_DB_RATION_DEFAULT 0.25 +#undef org_forstdb_SstFileManager_BYTES_MAX_DELETE_CHUNK_DEFAULT +#define org_forstdb_SstFileManager_BYTES_MAX_DELETE_CHUNK_DEFAULT 67108864LL +/* + * Class: org_forstdb_SstFileManager + * Method: newSstFileManager + * Signature: (JJJDJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstFileManager_newSstFileManager + (JNIEnv *, jclass, jlong, jlong, jlong, jdouble, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: setMaxAllowedSpaceUsage + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileManager_setMaxAllowedSpaceUsage + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: setCompactionBufferSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileManager_setCompactionBufferSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: isMaxAllowedSpaceReached + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_SstFileManager_isMaxAllowedSpaceReached + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: isMaxAllowedSpaceReachedIncludingCompactions + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_SstFileManager_isMaxAllowedSpaceReachedIncludingCompactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: getTotalSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstFileManager_getTotalSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: getTrackedFiles + * Signature: (J)Ljava/util/Map; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_SstFileManager_getTrackedFiles + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: getDeleteRateBytesPerSecond + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstFileManager_getDeleteRateBytesPerSecond + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: setDeleteRateBytesPerSecond + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileManager_setDeleteRateBytesPerSecond + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: getMaxTrashDBRatio + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_forstdb_SstFileManager_getMaxTrashDBRatio + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileManager + * Method: setMaxTrashDBRatio + * Signature: (JD)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileManager_setMaxTrashDBRatio + (JNIEnv *, jobject, jlong, jdouble); + +/* + * Class: org_forstdb_SstFileManager + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileManager_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_SstFileReader.h b/java/include/org_forstdb_SstFileReader.h new file mode 100644 index 000000000..688f87a4e --- /dev/null +++ b/java/include/org_forstdb_SstFileReader.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_SstFileReader */ + +#ifndef _Included_org_forstdb_SstFileReader +#define _Included_org_forstdb_SstFileReader +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_SstFileReader + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReader_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReader + * Method: newIterator + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstFileReader_newIterator + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_SstFileReader + * Method: open + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReader_open + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_SstFileReader + * Method: newSstFileReader + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstFileReader_newSstFileReader + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_SstFileReader + * Method: verifyChecksum + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReader_verifyChecksum + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReader + * Method: getTableProperties + * Signature: (J)Lorg/forstdb/TableProperties; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_SstFileReader_getTableProperties + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_SstFileReaderIterator.h b/java/include/org_forstdb_SstFileReaderIterator.h new file mode 100644 index 000000000..e8fde1efb --- /dev/null +++ b/java/include/org_forstdb_SstFileReaderIterator.h @@ -0,0 +1,173 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_SstFileReaderIterator */ + +#ifndef _Included_org_forstdb_SstFileReaderIterator +#define _Included_org_forstdb_SstFileReaderIterator +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: isValid0 + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_SstFileReaderIterator_isValid0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: seekToFirst0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_seekToFirst0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: seekToLast0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_seekToLast0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: next0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_next0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: prev0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_prev0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: refresh0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_refresh0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: seek0 + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_seek0 + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: seekForPrev0 + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_seekForPrev0 + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: status0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_status0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: seekDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_seekDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: seekForPrevDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_seekForPrevDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: seekByteArray0 + * Signature: (J[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_seekByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: seekForPrevByteArray0 + * Signature: (J[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileReaderIterator_seekForPrevByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: key0 + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_SstFileReaderIterator_key0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: value0 + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_SstFileReaderIterator_value0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: keyDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_SstFileReaderIterator_keyDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: keyByteArray0 + * Signature: (J[BII)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_SstFileReaderIterator_keyByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: valueDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_SstFileReaderIterator_valueDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_SstFileReaderIterator + * Method: valueByteArray0 + * Signature: (J[BII)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_SstFileReaderIterator_valueByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_SstFileWriter.h b/java/include/org_forstdb_SstFileWriter.h new file mode 100644 index 000000000..58af1dd58 --- /dev/null +++ b/java/include/org_forstdb_SstFileWriter.h @@ -0,0 +1,117 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_SstFileWriter */ + +#ifndef _Included_org_forstdb_SstFileWriter +#define _Included_org_forstdb_SstFileWriter +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_SstFileWriter + * Method: newSstFileWriter + * Signature: (JJJB)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstFileWriter_newSstFileWriter__JJJB + (JNIEnv *, jclass, jlong, jlong, jlong, jbyte); + +/* + * Class: org_forstdb_SstFileWriter + * Method: newSstFileWriter + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstFileWriter_newSstFileWriter__JJ + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_SstFileWriter + * Method: open + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_open + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_SstFileWriter + * Method: put + * Signature: (JJJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_put__JJJ + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_SstFileWriter + * Method: put + * Signature: (J[B[B)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_put__J_3B_3B + (JNIEnv *, jobject, jlong, jbyteArray, jbyteArray); + +/* + * Class: org_forstdb_SstFileWriter + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_putDirect + (JNIEnv *, jobject, jlong, jobject, jint, jint, jobject, jint, jint); + +/* + * Class: org_forstdb_SstFileWriter + * Method: fileSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstFileWriter_fileSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileWriter + * Method: merge + * Signature: (JJJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_merge__JJJ + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_SstFileWriter + * Method: merge + * Signature: (J[B[B)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_merge__J_3B_3B + (JNIEnv *, jobject, jlong, jbyteArray, jbyteArray); + +/* + * Class: org_forstdb_SstFileWriter + * Method: delete + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_delete__JJ + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_SstFileWriter + * Method: delete + * Signature: (J[B)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_delete__J_3B + (JNIEnv *, jobject, jlong, jbyteArray); + +/* + * Class: org_forstdb_SstFileWriter + * Method: finish + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_finish + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_SstFileWriter + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstFileWriter_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_SstPartitionerFixedPrefixFactory.h b/java/include/org_forstdb_SstPartitionerFixedPrefixFactory.h new file mode 100644 index 000000000..13b7db72e --- /dev/null +++ b/java/include/org_forstdb_SstPartitionerFixedPrefixFactory.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_SstPartitionerFixedPrefixFactory */ + +#ifndef _Included_org_forstdb_SstPartitionerFixedPrefixFactory +#define _Included_org_forstdb_SstPartitionerFixedPrefixFactory +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_SstPartitionerFixedPrefixFactory + * Method: newSstPartitionerFixedPrefixFactory0 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_SstPartitionerFixedPrefixFactory_newSstPartitionerFixedPrefixFactory0 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_SstPartitionerFixedPrefixFactory + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_SstPartitionerFixedPrefixFactory_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Statistics.h b/java/include/org_forstdb_Statistics.h new file mode 100644 index 000000000..de20acdc6 --- /dev/null +++ b/java/include/org_forstdb_Statistics.h @@ -0,0 +1,117 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Statistics */ + +#ifndef _Included_org_forstdb_Statistics +#define _Included_org_forstdb_Statistics +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Statistics + * Method: newStatistics + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Statistics_newStatistics__ + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_Statistics + * Method: newStatistics + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Statistics_newStatistics__J + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_Statistics + * Method: newStatistics + * Signature: ([B)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Statistics_newStatistics___3B + (JNIEnv *, jclass, jbyteArray); + +/* + * Class: org_forstdb_Statistics + * Method: newStatistics + * Signature: ([BJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Statistics_newStatistics___3BJ + (JNIEnv *, jclass, jbyteArray, jlong); + +/* + * Class: org_forstdb_Statistics + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Statistics_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Statistics + * Method: statsLevel + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Statistics_statsLevel + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Statistics + * Method: setStatsLevel + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Statistics_setStatsLevel + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Statistics + * Method: getTickerCount + * Signature: (JB)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Statistics_getTickerCount + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Statistics + * Method: getAndResetTickerCount + * Signature: (JB)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Statistics_getAndResetTickerCount + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Statistics + * Method: getHistogramData + * Signature: (JB)Lorg/forstdb/HistogramData; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_Statistics_getHistogramData + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Statistics + * Method: getHistogramString + * Signature: (JB)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_Statistics_getHistogramString + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_Statistics + * Method: reset + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Statistics_reset + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Statistics + * Method: toString + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_Statistics_toString + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_StringAppendOperator.h b/java/include/org_forstdb_StringAppendOperator.h new file mode 100644 index 000000000..b4a7fa77c --- /dev/null +++ b/java/include/org_forstdb_StringAppendOperator.h @@ -0,0 +1,37 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_StringAppendOperator */ + +#ifndef _Included_org_forstdb_StringAppendOperator +#define _Included_org_forstdb_StringAppendOperator +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_StringAppendOperator + * Method: newSharedStringAppendOperator + * Signature: (C)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_StringAppendOperator_newSharedStringAppendOperator__C + (JNIEnv *, jclass, jchar); + +/* + * Class: org_forstdb_StringAppendOperator + * Method: newSharedStringAppendOperator + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_StringAppendOperator_newSharedStringAppendOperator__Ljava_lang_String_2 + (JNIEnv *, jclass, jstring); + +/* + * Class: org_forstdb_StringAppendOperator + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_StringAppendOperator_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_ThreadStatus.h b/java/include/org_forstdb_ThreadStatus.h new file mode 100644 index 000000000..6c358e4e2 --- /dev/null +++ b/java/include/org_forstdb_ThreadStatus.h @@ -0,0 +1,69 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_ThreadStatus */ + +#ifndef _Included_org_forstdb_ThreadStatus +#define _Included_org_forstdb_ThreadStatus +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_ThreadStatus + * Method: getThreadTypeName + * Signature: (B)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ThreadStatus_getThreadTypeName + (JNIEnv *, jclass, jbyte); + +/* + * Class: org_forstdb_ThreadStatus + * Method: getOperationName + * Signature: (B)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ThreadStatus_getOperationName + (JNIEnv *, jclass, jbyte); + +/* + * Class: org_forstdb_ThreadStatus + * Method: microsToStringNative + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ThreadStatus_microsToStringNative + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_ThreadStatus + * Method: getOperationStageName + * Signature: (B)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ThreadStatus_getOperationStageName + (JNIEnv *, jclass, jbyte); + +/* + * Class: org_forstdb_ThreadStatus + * Method: getOperationPropertyName + * Signature: (BI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ThreadStatus_getOperationPropertyName + (JNIEnv *, jclass, jbyte, jint); + +/* + * Class: org_forstdb_ThreadStatus + * Method: interpretOperationProperties + * Signature: (B[J)Ljava/util/Map; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_ThreadStatus_interpretOperationProperties + (JNIEnv *, jclass, jbyte, jlongArray); + +/* + * Class: org_forstdb_ThreadStatus + * Method: getStateName + * Signature: (B)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_ThreadStatus_getStateName + (JNIEnv *, jclass, jbyte); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_TimedEnv.h b/java/include/org_forstdb_TimedEnv.h new file mode 100644 index 000000000..9fbc7ae94 --- /dev/null +++ b/java/include/org_forstdb_TimedEnv.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_TimedEnv */ + +#ifndef _Included_org_forstdb_TimedEnv +#define _Included_org_forstdb_TimedEnv +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_TimedEnv + * Method: createTimedEnv + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TimedEnv_createTimedEnv + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_TimedEnv + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TimedEnv_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_Transaction.h b/java/include/org_forstdb_Transaction.h new file mode 100644 index 000000000..eeb9dc73e --- /dev/null +++ b/java/include/org_forstdb_Transaction.h @@ -0,0 +1,613 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_Transaction */ + +#ifndef _Included_org_forstdb_Transaction +#define _Included_org_forstdb_Transaction +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_Transaction + * Method: setSnapshot + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_setSnapshot + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: setSnapshotOnNextOperation + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_setSnapshotOnNextOperation__J + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: setSnapshotOnNextOperation + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_setSnapshotOnNextOperation__JJ + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getSnapshot + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getSnapshot + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: clearSnapshot + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_clearSnapshot + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: prepare + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_prepare + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: commit + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_commit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: rollback + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_rollback + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: setSavePoint + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_setSavePoint + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: rollbackToSavePoint + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_rollbackToSavePoint + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: get + * Signature: (JJ[BIIJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_Transaction_get__JJ_3BIIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: get + * Signature: (JJ[BII[BIIJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Transaction_get__JJ_3BII_3BIIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getDirect + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Transaction_getDirect + (JNIEnv *, jobject, jlong, jlong, jobject, jint, jint, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: multiGet + * Signature: (JJ[[B[J)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_Transaction_multiGet__JJ_3_3B_3J + (JNIEnv *, jobject, jlong, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_Transaction + * Method: multiGet + * Signature: (JJ[[B)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_Transaction_multiGet__JJ_3_3B + (JNIEnv *, jobject, jlong, jlong, jobjectArray); + +/* + * Class: org_forstdb_Transaction + * Method: getForUpdate + * Signature: (JJ[BIIJZZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_Transaction_getForUpdate__JJ_3BIIJZZ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jlong, jboolean, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: getForUpdate + * Signature: (JJ[BII[BIIJZZ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Transaction_getForUpdate__JJ_3BII_3BIIJZZ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong, jboolean, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: getDirectForUpdate + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJZZ)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_Transaction_getDirectForUpdate + (JNIEnv *, jobject, jlong, jlong, jobject, jint, jint, jobject, jint, jint, jlong, jboolean, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: multiGetForUpdate + * Signature: (JJ[[B[J)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_Transaction_multiGetForUpdate__JJ_3_3B_3J + (JNIEnv *, jobject, jlong, jlong, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_Transaction + * Method: multiGetForUpdate + * Signature: (JJ[[B)[[B + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_Transaction_multiGetForUpdate__JJ_3_3B + (JNIEnv *, jobject, jlong, jlong, jobjectArray); + +/* + * Class: org_forstdb_Transaction + * Method: getIterator + * Signature: (JJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getIterator + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: put + * Signature: (J[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_put__J_3BII_3BII + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_Transaction + * Method: put + * Signature: (J[BII[BIIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_put__J_3BII_3BIIJZ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: put + * Signature: (J[[BI[[BIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_put__J_3_3BI_3_3BIJZ + (JNIEnv *, jobject, jlong, jobjectArray, jint, jobjectArray, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: put + * Signature: (J[[BI[[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_put__J_3_3BI_3_3BI + (JNIEnv *, jobject, jlong, jobjectArray, jint, jobjectArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_putDirect__JLjava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2IIJZ + (JNIEnv *, jobject, jlong, jobject, jint, jint, jobject, jint, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_putDirect__JLjava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2II + (JNIEnv *, jobject, jlong, jobject, jint, jint, jobject, jint, jint); + +/* + * Class: org_forstdb_Transaction + * Method: merge + * Signature: (J[BII[BIIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_merge__J_3BII_3BIIJZ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: mergeDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_mergeDirect__JLjava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2IIJZ + (JNIEnv *, jobject, jlong, jobject, jint, jint, jobject, jint, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: mergeDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_mergeDirect__JLjava_nio_ByteBuffer_2IILjava_nio_ByteBuffer_2II + (JNIEnv *, jobject, jlong, jobject, jint, jint, jobject, jint, jint); + +/* + * Class: org_forstdb_Transaction + * Method: merge + * Signature: (J[BII[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_merge__J_3BII_3BII + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_Transaction + * Method: delete + * Signature: (J[BIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_delete__J_3BIJZ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: delete + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_delete__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: delete + * Signature: (J[[BIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_delete__J_3_3BIJZ + (JNIEnv *, jobject, jlong, jobjectArray, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: delete + * Signature: (J[[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_delete__J_3_3BI + (JNIEnv *, jobject, jlong, jobjectArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: singleDelete + * Signature: (J[BIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_singleDelete__J_3BIJZ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: singleDelete + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_singleDelete__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: singleDelete + * Signature: (J[[BIJZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_singleDelete__J_3_3BIJZ + (JNIEnv *, jobject, jlong, jobjectArray, jint, jlong, jboolean); + +/* + * Class: org_forstdb_Transaction + * Method: singleDelete + * Signature: (J[[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_singleDelete__J_3_3BI + (JNIEnv *, jobject, jlong, jobjectArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: putUntracked + * Signature: (J[BI[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_putUntracked__J_3BI_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: putUntracked + * Signature: (J[BI[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_putUntracked__J_3BI_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: putUntracked + * Signature: (J[[BI[[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_putUntracked__J_3_3BI_3_3BIJ + (JNIEnv *, jobject, jlong, jobjectArray, jint, jobjectArray, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: putUntracked + * Signature: (J[[BI[[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_putUntracked__J_3_3BI_3_3BI + (JNIEnv *, jobject, jlong, jobjectArray, jint, jobjectArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: mergeUntracked + * Signature: (J[BII[BIIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_mergeUntracked + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint, jbyteArray, jint, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: mergeUntrackedDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_mergeUntrackedDirect + (JNIEnv *, jobject, jlong, jobject, jint, jint, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: deleteUntracked + * Signature: (J[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_deleteUntracked__J_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: deleteUntracked + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_deleteUntracked__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: deleteUntracked + * Signature: (J[[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_deleteUntracked__J_3_3BIJ + (JNIEnv *, jobject, jlong, jobjectArray, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: deleteUntracked + * Signature: (J[[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_deleteUntracked__J_3_3BI + (JNIEnv *, jobject, jlong, jobjectArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: putLogData + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_putLogData + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: disableIndexing + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_disableIndexing + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: enableIndexing + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_enableIndexing + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getNumKeys + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getNumKeys + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getNumPuts + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getNumPuts + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getNumDeletes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getNumDeletes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getNumMerges + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getNumMerges + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getElapsedTime + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getElapsedTime + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getWriteBatch + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getWriteBatch + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: setLockTimeout + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_setLockTimeout + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getWriteOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getWriteOptions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: setWriteOptions + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_setWriteOptions + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: undoGetForUpdate + * Signature: (J[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_undoGetForUpdate__J_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: undoGetForUpdate + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_undoGetForUpdate__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_Transaction + * Method: rebuildFromWriteBatch + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_rebuildFromWriteBatch + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getCommitTimeWriteBatch + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getCommitTimeWriteBatch + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: setLogNumber + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_setLogNumber + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getLogNumber + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getLogNumber + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: setName + * Signature: (JLjava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_setName + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_Transaction + * Method: getName + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_forstdb_Transaction_getName + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getID + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getID + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: isDeadlockDetect + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_Transaction_isDeadlockDetect + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getWaitingTxns + * Signature: (J)Lorg/forstdb/Transaction/WaitingTransactions; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_Transaction_getWaitingTxns + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getState + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_Transaction_getState + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: getId + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_Transaction_getId + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_Transaction + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_Transaction_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_TransactionDB.h b/java/include/org_forstdb_TransactionDB.h new file mode 100644 index 000000000..6e71740dd --- /dev/null +++ b/java/include/org_forstdb_TransactionDB.h @@ -0,0 +1,119 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_TransactionDB */ + +#ifndef _Included_org_forstdb_TransactionDB +#define _Included_org_forstdb_TransactionDB +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_TransactionDB_NOT_FOUND +#define org_forstdb_TransactionDB_NOT_FOUND -1L +/* + * Class: org_forstdb_TransactionDB + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDB_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: open + * Signature: (JJLjava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDB_open__JJLjava_lang_String_2 + (JNIEnv *, jclass, jlong, jlong, jstring); + +/* + * Class: org_forstdb_TransactionDB + * Method: open + * Signature: (JJLjava/lang/String;[[B[J)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_TransactionDB_open__JJLjava_lang_String_2_3_3B_3J + (JNIEnv *, jclass, jlong, jlong, jstring, jobjectArray, jlongArray); + +/* + * Class: org_forstdb_TransactionDB + * Method: closeDatabase + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDB_closeDatabase + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: beginTransaction + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDB_beginTransaction__JJ + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: beginTransaction + * Signature: (JJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDB_beginTransaction__JJJ + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: beginTransaction_withOld + * Signature: (JJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDB_beginTransaction_1withOld__JJJ + (JNIEnv *, jobject, jlong, jlong, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: beginTransaction_withOld + * Signature: (JJJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDB_beginTransaction_1withOld__JJJJ + (JNIEnv *, jobject, jlong, jlong, jlong, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: getTransactionByName + * Signature: (JLjava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDB_getTransactionByName + (JNIEnv *, jobject, jlong, jstring); + +/* + * Class: org_forstdb_TransactionDB + * Method: getAllPreparedTransactions + * Signature: (J)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_TransactionDB_getAllPreparedTransactions + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: getLockStatusData + * Signature: (J)Ljava/util/Map; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_TransactionDB_getLockStatusData + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: getDeadlockInfoBuffer + * Signature: (J)[Lorg/forstdb/TransactionDB/DeadlockPath; + */ +JNIEXPORT jobjectArray JNICALL Java_org_forstdb_TransactionDB_getDeadlockInfoBuffer + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDB + * Method: setDeadlockInfoBufferSize + * Signature: (JI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDB_setDeadlockInfoBufferSize + (JNIEnv *, jobject, jlong, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_TransactionDBOptions.h b/java/include/org_forstdb_TransactionDBOptions.h new file mode 100644 index 000000000..2fd6def68 --- /dev/null +++ b/java/include/org_forstdb_TransactionDBOptions.h @@ -0,0 +1,109 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_TransactionDBOptions */ + +#ifndef _Included_org_forstdb_TransactionDBOptions +#define _Included_org_forstdb_TransactionDBOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_TransactionDBOptions + * Method: newTransactionDBOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDBOptions_newTransactionDBOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: getMaxNumLocks + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDBOptions_getMaxNumLocks + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: setMaxNumLocks + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDBOptions_setMaxNumLocks + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: getNumStripes + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDBOptions_getNumStripes + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: setNumStripes + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDBOptions_setNumStripes + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: getTransactionLockTimeout + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDBOptions_getTransactionLockTimeout + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: setTransactionLockTimeout + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDBOptions_setTransactionLockTimeout + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: getDefaultLockTimeout + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionDBOptions_getDefaultLockTimeout + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: setDefaultLockTimeout + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDBOptions_setDefaultLockTimeout + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: getWritePolicy + * Signature: (J)B + */ +JNIEXPORT jbyte JNICALL Java_org_forstdb_TransactionDBOptions_getWritePolicy + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: setWritePolicy + * Signature: (JB)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDBOptions_setWritePolicy + (JNIEnv *, jobject, jlong, jbyte); + +/* + * Class: org_forstdb_TransactionDBOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionDBOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_TransactionLogIterator.h b/java/include/org_forstdb_TransactionLogIterator.h new file mode 100644 index 000000000..ee8c79d99 --- /dev/null +++ b/java/include/org_forstdb_TransactionLogIterator.h @@ -0,0 +1,53 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_TransactionLogIterator */ + +#ifndef _Included_org_forstdb_TransactionLogIterator +#define _Included_org_forstdb_TransactionLogIterator +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_TransactionLogIterator + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionLogIterator_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionLogIterator + * Method: isValid + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_TransactionLogIterator_isValid + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionLogIterator + * Method: next + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionLogIterator_next + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionLogIterator + * Method: status + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionLogIterator_status + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionLogIterator + * Method: getBatch + * Signature: (J)Lorg/forstdb/TransactionLogIterator/BatchResult; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_TransactionLogIterator_getBatch + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_TransactionOptions.h b/java/include/org_forstdb_TransactionOptions.h new file mode 100644 index 000000000..673a41c5f --- /dev/null +++ b/java/include/org_forstdb_TransactionOptions.h @@ -0,0 +1,125 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_TransactionOptions */ + +#ifndef _Included_org_forstdb_TransactionOptions +#define _Included_org_forstdb_TransactionOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_TransactionOptions + * Method: newTransactionOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionOptions_newTransactionOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_TransactionOptions + * Method: isSetSnapshot + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_TransactionOptions_isSetSnapshot + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: setSetSnapshot + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionOptions_setSetSnapshot + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_TransactionOptions + * Method: isDeadlockDetect + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_TransactionOptions_isDeadlockDetect + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: setDeadlockDetect + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionOptions_setDeadlockDetect + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_TransactionOptions + * Method: getLockTimeout + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionOptions_getLockTimeout + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: setLockTimeout + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionOptions_setLockTimeout + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: getExpiration + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionOptions_getExpiration + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: setExpiration + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionOptions_setExpiration + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: getDeadlockDetectDepth + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionOptions_getDeadlockDetectDepth + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: setDeadlockDetectDepth + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionOptions_setDeadlockDetectDepth + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: getMaxWriteBatchSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TransactionOptions_getMaxWriteBatchSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: setMaxWriteBatchSize + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionOptions_setMaxWriteBatchSize + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_TransactionOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TransactionOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_TtlDB.h b/java/include/org_forstdb_TtlDB.h new file mode 100644 index 000000000..9f77960ed --- /dev/null +++ b/java/include/org_forstdb_TtlDB.h @@ -0,0 +1,55 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_TtlDB */ + +#ifndef _Included_org_forstdb_TtlDB +#define _Included_org_forstdb_TtlDB +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_TtlDB_NOT_FOUND +#define org_forstdb_TtlDB_NOT_FOUND -1L +/* + * Class: org_forstdb_TtlDB + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TtlDB_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_TtlDB + * Method: open + * Signature: (JLjava/lang/String;IZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TtlDB_open + (JNIEnv *, jclass, jlong, jstring, jint, jboolean); + +/* + * Class: org_forstdb_TtlDB + * Method: openCF + * Signature: (JLjava/lang/String;[[B[J[IZ)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_TtlDB_openCF + (JNIEnv *, jclass, jlong, jstring, jobjectArray, jlongArray, jintArray, jboolean); + +/* + * Class: org_forstdb_TtlDB + * Method: createColumnFamilyWithTtl + * Signature: (J[BJI)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_TtlDB_createColumnFamilyWithTtl + (JNIEnv *, jobject, jlong, jbyteArray, jlong, jint); + +/* + * Class: org_forstdb_TtlDB + * Method: closeDatabase + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_TtlDB_closeDatabase + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_UInt64AddOperator.h b/java/include/org_forstdb_UInt64AddOperator.h new file mode 100644 index 000000000..930b61362 --- /dev/null +++ b/java/include/org_forstdb_UInt64AddOperator.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_UInt64AddOperator */ + +#ifndef _Included_org_forstdb_UInt64AddOperator +#define _Included_org_forstdb_UInt64AddOperator +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_UInt64AddOperator + * Method: newSharedUInt64AddOperator + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_UInt64AddOperator_newSharedUInt64AddOperator + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_UInt64AddOperator + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_UInt64AddOperator_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_VectorMemTableConfig.h b/java/include/org_forstdb_VectorMemTableConfig.h new file mode 100644 index 000000000..b25ed0fbb --- /dev/null +++ b/java/include/org_forstdb_VectorMemTableConfig.h @@ -0,0 +1,23 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_VectorMemTableConfig */ + +#ifndef _Included_org_forstdb_VectorMemTableConfig +#define _Included_org_forstdb_VectorMemTableConfig +#ifdef __cplusplus +extern "C" { +#endif +#undef org_forstdb_VectorMemTableConfig_DEFAULT_RESERVED_SIZE +#define org_forstdb_VectorMemTableConfig_DEFAULT_RESERVED_SIZE 0L +/* + * Class: org_forstdb_VectorMemTableConfig + * Method: newMemTableFactoryHandle + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_VectorMemTableConfig_newMemTableFactoryHandle + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_WBWIRocksIterator.h b/java/include/org_forstdb_WBWIRocksIterator.h new file mode 100644 index 000000000..d42e5b6b8 --- /dev/null +++ b/java/include/org_forstdb_WBWIRocksIterator.h @@ -0,0 +1,133 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_WBWIRocksIterator */ + +#ifndef _Included_org_forstdb_WBWIRocksIterator +#define _Included_org_forstdb_WBWIRocksIterator +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: isValid0 + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WBWIRocksIterator_isValid0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: seekToFirst0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_seekToFirst0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: seekToLast0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_seekToLast0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: next0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_next0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: prev0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_prev0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: refresh0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_refresh0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: seek0 + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_seek0 + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: seekForPrev0 + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_seekForPrev0 + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: status0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_status0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: seekDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_seekDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: seekForPrevDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_seekForPrevDirect0 + (JNIEnv *, jobject, jlong, jobject, jint, jint); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: seekByteArray0 + * Signature: (J[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_seekByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: seekForPrevByteArray0 + * Signature: (J[BII)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WBWIRocksIterator_seekForPrevByteArray0 + (JNIEnv *, jobject, jlong, jbyteArray, jint, jint); + +/* + * Class: org_forstdb_WBWIRocksIterator + * Method: entry1 + * Signature: (J)[J + */ +JNIEXPORT jlongArray JNICALL Java_org_forstdb_WBWIRocksIterator_entry1 + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_WriteBatch.h b/java/include/org_forstdb_WriteBatch.h new file mode 100644 index 000000000..b485ce83a --- /dev/null +++ b/java/include/org_forstdb_WriteBatch.h @@ -0,0 +1,301 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_WriteBatch */ + +#ifndef _Included_org_forstdb_WriteBatch +#define _Included_org_forstdb_WriteBatch +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_WriteBatch + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: count0 + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_WriteBatch_count0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: put + * Signature: (J[BI[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_put__J_3BI_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatch + * Method: put + * Signature: (J[BI[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_put__J_3BI_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_putDirect + (JNIEnv *, jobject, jlong, jobject, jint, jint, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: merge + * Signature: (J[BI[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_merge__J_3BI_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatch + * Method: merge + * Signature: (J[BI[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_merge__J_3BI_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: delete + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_delete__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatch + * Method: delete + * Signature: (J[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_delete__J_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: singleDelete + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_singleDelete__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatch + * Method: singleDelete + * Signature: (J[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_singleDelete__J_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: deleteDirect + * Signature: (JLjava/nio/ByteBuffer;IIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_deleteDirect + (JNIEnv *, jobject, jlong, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: deleteRange + * Signature: (J[BI[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_deleteRange__J_3BI_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatch + * Method: deleteRange + * Signature: (J[BI[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_deleteRange__J_3BI_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: putLogData + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_putLogData + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatch + * Method: clear0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_clear0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: setSavePoint0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_setSavePoint0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: rollbackToSavePoint0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_rollbackToSavePoint0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: popSavePoint + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_popSavePoint + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: setMaxBytes + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_setMaxBytes + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: newWriteBatch + * Signature: (I)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatch_newWriteBatch__I + (JNIEnv *, jclass, jint); + +/* + * Class: org_forstdb_WriteBatch + * Method: newWriteBatch + * Signature: ([BI)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatch_newWriteBatch___3BI + (JNIEnv *, jclass, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatch + * Method: iterate + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_iterate + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: data + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_WriteBatch_data + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: getDataSize + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatch_getDataSize + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasPut + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasPut + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasDelete + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasDelete + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasSingleDelete + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasSingleDelete + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasDeleteRange + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasDeleteRange + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasMerge + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasMerge + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasBeginPrepare + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasBeginPrepare + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasEndPrepare + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasEndPrepare + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasCommit + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasCommit + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: hasRollback + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteBatch_hasRollback + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: markWalTerminationPoint + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatch_markWalTerminationPoint + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatch + * Method: getWalTerminationPoint + * Signature: (J)Lorg/forstdb/WriteBatch/SavePoint; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_WriteBatch_getWalTerminationPoint + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_WriteBatchTest.h b/java/include/org_forstdb_WriteBatchTest.h new file mode 100644 index 000000000..2bb6651d4 --- /dev/null +++ b/java/include/org_forstdb_WriteBatchTest.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_WriteBatchTest */ + +#ifndef _Included_org_forstdb_WriteBatchTest +#define _Included_org_forstdb_WriteBatchTest +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_WriteBatchTest + * Method: getContents + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_WriteBatchTest_getContents + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_WriteBatchTestInternalHelper.h b/java/include/org_forstdb_WriteBatchTestInternalHelper.h new file mode 100644 index 000000000..15d6e041f --- /dev/null +++ b/java/include/org_forstdb_WriteBatchTestInternalHelper.h @@ -0,0 +1,37 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_WriteBatchTestInternalHelper */ + +#ifndef _Included_org_forstdb_WriteBatchTestInternalHelper +#define _Included_org_forstdb_WriteBatchTestInternalHelper +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_WriteBatchTestInternalHelper + * Method: setSequence + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchTestInternalHelper_setSequence + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_forstdb_WriteBatchTestInternalHelper + * Method: sequence + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatchTestInternalHelper_sequence + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_WriteBatchTestInternalHelper + * Method: append + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchTestInternalHelper_append + (JNIEnv *, jclass, jlong, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_WriteBatchWithIndex.h b/java/include/org_forstdb_WriteBatchWithIndex.h new file mode 100644 index 000000000..a39427580 --- /dev/null +++ b/java/include/org_forstdb_WriteBatchWithIndex.h @@ -0,0 +1,261 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_WriteBatchWithIndex */ + +#ifndef _Included_org_forstdb_WriteBatchWithIndex +#define _Included_org_forstdb_WriteBatchWithIndex +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: count0 + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_forstdb_WriteBatchWithIndex_count0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: put + * Signature: (J[BI[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_put__J_3BI_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: put + * Signature: (J[BI[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_put__J_3BI_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_putDirect + (JNIEnv *, jobject, jlong, jobject, jint, jint, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: merge + * Signature: (J[BI[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_merge__J_3BI_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: merge + * Signature: (J[BI[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_merge__J_3BI_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: delete + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_delete__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: delete + * Signature: (J[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_delete__J_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: singleDelete + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_singleDelete__J_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: singleDelete + * Signature: (J[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_singleDelete__J_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: deleteDirect + * Signature: (JLjava/nio/ByteBuffer;IIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_deleteDirect + (JNIEnv *, jobject, jlong, jobject, jint, jint, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: deleteRange + * Signature: (J[BI[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_deleteRange__J_3BI_3BI + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: deleteRange + * Signature: (J[BI[BIJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_deleteRange__J_3BI_3BIJ + (JNIEnv *, jobject, jlong, jbyteArray, jint, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: putLogData + * Signature: (J[BI)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_putLogData + (JNIEnv *, jobject, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: clear0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_clear0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: setSavePoint0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_setSavePoint0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: rollbackToSavePoint0 + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_rollbackToSavePoint0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: popSavePoint + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_popSavePoint + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: setMaxBytes + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBatchWithIndex_setMaxBytes + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: getWriteBatch + * Signature: (J)Lorg/forstdb/WriteBatch; + */ +JNIEXPORT jobject JNICALL Java_org_forstdb_WriteBatchWithIndex_getWriteBatch + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: newWriteBatchWithIndex + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatchWithIndex_newWriteBatchWithIndex__ + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: newWriteBatchWithIndex + * Signature: (Z)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z + (JNIEnv *, jclass, jboolean); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: newWriteBatchWithIndex + * Signature: (JBIZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatchWithIndex_newWriteBatchWithIndex__JBIZ + (JNIEnv *, jclass, jlong, jbyte, jint, jboolean); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: iterator0 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatchWithIndex_iterator0 + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: iterator1 + * Signature: (JJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatchWithIndex_iterator1 + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: iteratorWithBase + * Signature: (JJJJ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatchWithIndex_iteratorWithBase + (JNIEnv *, jobject, jlong, jlong, jlong, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: getFromBatch + * Signature: (JJ[BI)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_WriteBatchWithIndex_getFromBatch__JJ_3BI + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: getFromBatch + * Signature: (JJ[BIJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_WriteBatchWithIndex_getFromBatch__JJ_3BIJ + (JNIEnv *, jobject, jlong, jlong, jbyteArray, jint, jlong); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: getFromBatchAndDB + * Signature: (JJJ[BI)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BI + (JNIEnv *, jobject, jlong, jlong, jlong, jbyteArray, jint); + +/* + * Class: org_forstdb_WriteBatchWithIndex + * Method: getFromBatchAndDB + * Signature: (JJJ[BIJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_forstdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BIJ + (JNIEnv *, jobject, jlong, jlong, jlong, jbyteArray, jint, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_WriteBatch_Handler.h b/java/include/org_forstdb_WriteBatch_Handler.h new file mode 100644 index 000000000..1015031f2 --- /dev/null +++ b/java/include/org_forstdb_WriteBatch_Handler.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_WriteBatch_Handler */ + +#ifndef _Included_org_forstdb_WriteBatch_Handler +#define _Included_org_forstdb_WriteBatch_Handler +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_WriteBatch_Handler + * Method: createNewHandler0 + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBatch_00024Handler_createNewHandler0 + (JNIEnv *, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_WriteBufferManager.h b/java/include/org_forstdb_WriteBufferManager.h new file mode 100644 index 000000000..0af6a74bd --- /dev/null +++ b/java/include/org_forstdb_WriteBufferManager.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_WriteBufferManager */ + +#ifndef _Included_org_forstdb_WriteBufferManager +#define _Included_org_forstdb_WriteBufferManager +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_WriteBufferManager + * Method: newWriteBufferManager + * Signature: (JJZ)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteBufferManager_newWriteBufferManager + (JNIEnv *, jclass, jlong, jlong, jboolean); + +/* + * Class: org_forstdb_WriteBufferManager + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteBufferManager_disposeInternal + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_WriteOptions.h b/java/include/org_forstdb_WriteOptions.h new file mode 100644 index 000000000..01ecfa9df --- /dev/null +++ b/java/include/org_forstdb_WriteOptions.h @@ -0,0 +1,133 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_WriteOptions */ + +#ifndef _Included_org_forstdb_WriteOptions +#define _Included_org_forstdb_WriteOptions +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_WriteOptions + * Method: newWriteOptions + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteOptions_newWriteOptions + (JNIEnv *, jclass); + +/* + * Class: org_forstdb_WriteOptions + * Method: copyWriteOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_forstdb_WriteOptions_copyWriteOptions + (JNIEnv *, jclass, jlong); + +/* + * Class: org_forstdb_WriteOptions + * Method: disposeInternal + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteOptions_disposeInternal + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteOptions + * Method: setSync + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteOptions_setSync + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_WriteOptions + * Method: sync + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteOptions_sync + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteOptions + * Method: setDisableWAL + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteOptions_setDisableWAL + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_WriteOptions + * Method: disableWAL + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteOptions_disableWAL + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteOptions + * Method: setIgnoreMissingColumnFamilies + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteOptions_setIgnoreMissingColumnFamilies + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_WriteOptions + * Method: ignoreMissingColumnFamilies + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteOptions_ignoreMissingColumnFamilies + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteOptions + * Method: setNoSlowdown + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteOptions_setNoSlowdown + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_WriteOptions + * Method: noSlowdown + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteOptions_noSlowdown + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteOptions + * Method: setLowPri + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteOptions_setLowPri + (JNIEnv *, jobject, jlong, jboolean); + +/* + * Class: org_forstdb_WriteOptions + * Method: lowPri + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteOptions_lowPri + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteOptions + * Method: memtableInsertHintPerBatch + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_forstdb_WriteOptions_memtableInsertHintPerBatch + (JNIEnv *, jobject, jlong); + +/* + * Class: org_forstdb_WriteOptions + * Method: setMemtableInsertHintPerBatch + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_WriteOptions_setMemtableInsertHintPerBatch + (JNIEnv *, jobject, jlong, jboolean); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/include/org_forstdb_test_TestableEventListener.h b/java/include/org_forstdb_test_TestableEventListener.h new file mode 100644 index 000000000..4e9d36df5 --- /dev/null +++ b/java/include/org_forstdb_test_TestableEventListener.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_forstdb_test_TestableEventListener */ + +#ifndef _Included_org_forstdb_test_TestableEventListener +#define _Included_org_forstdb_test_TestableEventListener +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_forstdb_test_TestableEventListener + * Method: invokeAllCallbacks + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_forstdb_test_TestableEventListener_invokeAllCallbacks + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/java/jmh/pom.xml b/java/jmh/pom.xml index 3016aefa7..6c606f6bd 100644 --- a/java/jmh/pom.xml +++ b/java/jmh/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.rocksdb + org.forstdb rocksdbjni-jmh 1.0-SNAPSHOT @@ -48,7 +48,7 @@ - org.rocksdb + org.forstdb rocksdbjni 7.9.0-SNAPSHOT diff --git a/java/jmh/src/main/java/org/rocksdb/jmh/ComparatorBenchmarks.java b/java/jmh/src/main/java/org/rocksdb/jmh/ComparatorBenchmarks.java index 1973b5487..e0c9a437c 100644 --- a/java/jmh/src/main/java/org/rocksdb/jmh/ComparatorBenchmarks.java +++ b/java/jmh/src/main/java/org/rocksdb/jmh/ComparatorBenchmarks.java @@ -4,20 +4,20 @@ * COPYING file in the root directory) and Apache 2.0 License * (found in the LICENSE.Apache file in the root directory). */ -package org.rocksdb.jmh; +package org.forstdb.jmh; import org.openjdk.jmh.annotations.*; -import org.rocksdb.*; -import org.rocksdb.util.BytewiseComparator; -import org.rocksdb.util.FileUtils; -import org.rocksdb.util.ReverseBytewiseComparator; +import org.forstdb.*; +import org.forstdb.util.BytewiseComparator; +import org.forstdb.util.FileUtils; +import org.forstdb.util.ReverseBytewiseComparator; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicInteger; -import static org.rocksdb.util.KVUtils.ba; +import static org.forstdb.util.KVUtils.ba; @State(Scope.Benchmark) public class ComparatorBenchmarks { diff --git a/java/jmh/src/main/java/org/rocksdb/jmh/GetBenchmarks.java b/java/jmh/src/main/java/org/rocksdb/jmh/GetBenchmarks.java index 1c4329b3a..6155585e2 100644 --- a/java/jmh/src/main/java/org/rocksdb/jmh/GetBenchmarks.java +++ b/java/jmh/src/main/java/org/rocksdb/jmh/GetBenchmarks.java @@ -4,9 +4,9 @@ * COPYING file in the root directory) and Apache 2.0 License * (found in the LICENSE.Apache file in the root directory). */ -package org.rocksdb.jmh; +package org.forstdb.jmh; -import static org.rocksdb.util.KVUtils.ba; +import static org.forstdb.util.KVUtils.ba; import java.io.IOException; import java.nio.ByteBuffer; @@ -17,8 +17,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.openjdk.jmh.annotations.*; -import org.rocksdb.*; -import org.rocksdb.util.FileUtils; +import org.forstdb.*; +import org.forstdb.util.FileUtils; @State(Scope.Benchmark) public class GetBenchmarks { diff --git a/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java b/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java index d37447716..933906cde 100644 --- a/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java +++ b/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java @@ -4,10 +4,10 @@ * COPYING file in the root directory) and Apache 2.0 License * (found in the LICENSE.Apache file in the root directory). */ -package org.rocksdb.jmh; +package org.forstdb.jmh; -import static org.rocksdb.util.KVUtils.ba; -import static org.rocksdb.util.KVUtils.keys; +import static org.forstdb.util.KVUtils.ba; +import static org.forstdb.util.KVUtils.keys; import java.io.IOException; import java.nio.ByteBuffer; @@ -21,8 +21,8 @@ import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.OptionsBuilder; -import org.rocksdb.*; -import org.rocksdb.util.FileUtils; +import org.forstdb.*; +import org.forstdb.util.FileUtils; @State(Scope.Thread) public class MultiGetBenchmarks { diff --git a/java/jmh/src/main/java/org/rocksdb/jmh/PutBenchmarks.java b/java/jmh/src/main/java/org/rocksdb/jmh/PutBenchmarks.java index 5aae21cb9..705e57fb8 100644 --- a/java/jmh/src/main/java/org/rocksdb/jmh/PutBenchmarks.java +++ b/java/jmh/src/main/java/org/rocksdb/jmh/PutBenchmarks.java @@ -4,20 +4,21 @@ * COPYING file in the root directory) and Apache 2.0 License * (found in the LICENSE.Apache file in the root directory). */ -package org.rocksdb.jmh; +package org.forstdb.jmh; -import org.openjdk.jmh.annotations.*; -import org.rocksdb.*; -import org.rocksdb.util.FileUtils; +import static org.forstdb.util.KVUtils.ba; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; - -import static org.rocksdb.util.KVUtils.ba; +import org.openjdk.jmh.annotations.*; +import org.forstdb.*; +import org.forstdb.util.FileUtils; @State(Scope.Benchmark) public class PutBenchmarks { @@ -30,12 +31,24 @@ public class PutBenchmarks { }) String columnFamilyTestType; + @Param({"1000", "100000"}) int keyCount; + + @Param({"12", "64", "128"}) int keySize; + + @Param({"64", "1024", "65536"}) int valueSize; + + @Param({"16"}) int bufferListSize; + Path dbDir; DBOptions options; int cfs = 0; // number of column families private AtomicInteger cfHandlesIdx; ColumnFamilyHandle[] cfHandles; RocksDB db; + List keyBuffers = new ArrayList<>(bufferListSize); + List valueBuffers = new ArrayList<>(bufferListSize); + List keyBuffersBB = new ArrayList<>(bufferListSize); + List valueBuffersBB = new ArrayList<>(bufferListSize); @Setup(Level.Trial) public void setup() throws IOException, RocksDBException { @@ -68,6 +81,34 @@ public void setup() throws IOException, RocksDBException { final List cfHandlesList = new ArrayList<>(cfDescriptors.size()); db = RocksDB.open(options, dbDir.toAbsolutePath().toString(), cfDescriptors, cfHandlesList); cfHandles = cfHandlesList.toArray(new ColumnFamilyHandle[0]); + + for (int i = 0; i < bufferListSize; i++) { + final byte[] keyArr = new byte[keySize]; + Arrays.fill(keyArr, (byte) 0x30); + keyBuffers.add(keyArr); + } + + for (int i = 0; i < bufferListSize; i++) { + final byte[] valueArr = new byte[valueSize]; + Arrays.fill(valueArr, (byte) 0x30); + valueBuffers.add(valueArr); + } + + for (int i = 0; i < bufferListSize; i++) { + final ByteBuffer keyBB = ByteBuffer.allocateDirect(keySize); + byte[] keyArr = new byte[keySize]; + Arrays.fill(keyArr, (byte) 0x30); + keyBB.put(keyArr); + keyBuffersBB.add(keyBB); + } + + for (int i = 0; i < bufferListSize; i++) { + final ByteBuffer valueBB = ByteBuffer.allocateDirect(valueSize); + byte[] valueArr = new byte[valueSize]; + Arrays.fill(valueArr, (byte) 0x30); + valueBB.put(valueArr); + valueBuffersBB.add(valueBB); + } } @TearDown(Level.Trial) @@ -104,9 +145,79 @@ public int next() { } } + private T borrow(final List buffers) { + synchronized (buffers) { + while (true) { + if (buffers.isEmpty()) { + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + return null; + } + continue; + } + return buffers.remove(0); + } + } + } + + private void repay(final List buffers, final T buffer) { + synchronized (buffers) { + buffers.add(buffer); + } + } + + @Benchmark + public void put(final Counter counter) throws RocksDBException { + byte[] keyBuf = borrow(keyBuffers); + byte[] valueBuf = borrow(valueBuffers); + + final int i = counter.next(); + final byte[] keyPrefix = ba("key" + i); + final byte[] valuePrefix = ba("value" + i); + System.arraycopy(keyPrefix, 0, keyBuf, 0, keyPrefix.length); + System.arraycopy(valuePrefix, 0, valueBuf, 0, valuePrefix.length); + db.put(getColumnFamily(), keyBuf, valueBuf); + + repay(keyBuffers, keyBuf); + repay(valueBuffers, valueBuf); + } + + @Benchmark + public void putByteArrays(final Counter counter) throws RocksDBException { + byte[] keyBuf = borrow(keyBuffers); + byte[] valueBuf = borrow(valueBuffers); + + final int i = counter.next(); + final byte[] keyPrefix = ba("key" + i); + final byte[] valuePrefix = ba("value" + i); + System.arraycopy(keyPrefix, 0, keyBuf, 0, keyPrefix.length); + System.arraycopy(valuePrefix, 0, valueBuf, 0, valuePrefix.length); + db.put(getColumnFamily(), new WriteOptions(), keyBuf, valueBuf); + + repay(keyBuffers, keyBuf); + repay(valueBuffers, valueBuf); + } + @Benchmark - public void put(final ComparatorBenchmarks.Counter counter) throws RocksDBException { + public void putByteBuffers(final Counter counter) throws RocksDBException { + ByteBuffer keyBuf = borrow(keyBuffersBB); + keyBuf.clear(); + ByteBuffer valueBuf = borrow(valueBuffersBB); + valueBuf.clear(); + final int i = counter.next(); - db.put(getColumnFamily(), ba("key" + i), ba("value" + i)); + final byte[] keyPrefix = ba("key" + i); + final byte[] valuePrefix = ba("value" + i); + keyBuf.put(keyPrefix, 0, keyPrefix.length); + keyBuf.position(keySize); + keyBuf.flip(); + valueBuf.put(valuePrefix, 0, valuePrefix.length); + valueBuf.position(valueSize); + valueBuf.flip(); + db.put(getColumnFamily(), new WriteOptions(), keyBuf, valueBuf); + + repay(keyBuffersBB, keyBuf); + repay(valueBuffersBB, valueBuf); } } diff --git a/java/jmh/src/main/java/org/rocksdb/util/FileUtils.java b/java/jmh/src/main/java/org/rocksdb/util/FileUtils.java index 63744a14f..6c66f0c13 100644 --- a/java/jmh/src/main/java/org/rocksdb/util/FileUtils.java +++ b/java/jmh/src/main/java/org/rocksdb/util/FileUtils.java @@ -4,7 +4,7 @@ * COPYING file in the root directory) and Apache 2.0 License * (found in the LICENSE.Apache file in the root directory). */ -package org.rocksdb.util; +package org.forstdb.util; import java.io.IOException; import java.nio.file.FileVisitResult; diff --git a/java/jmh/src/main/java/org/rocksdb/util/KVUtils.java b/java/jmh/src/main/java/org/rocksdb/util/KVUtils.java index 5077291c8..a419ba78c 100644 --- a/java/jmh/src/main/java/org/rocksdb/util/KVUtils.java +++ b/java/jmh/src/main/java/org/rocksdb/util/KVUtils.java @@ -4,7 +4,7 @@ * COPYING file in the root directory) and Apache 2.0 License * (found in the LICENSE.Apache file in the root directory). */ -package org.rocksdb.util; +package org.forstdb.util; import static java.nio.charset.StandardCharsets.UTF_8; diff --git a/java/pmd-rules.xml b/java/pmd-rules.xml new file mode 100644 index 000000000..97ce03629 --- /dev/null +++ b/java/pmd-rules.xml @@ -0,0 +1,63 @@ + + + + + + Custom rules for checking RocksDB + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/pom.xml.template b/java/pom.xml.template index 0090ff142..52fabfc3e 100644 --- a/java/pom.xml.template +++ b/java/pom.xml.template @@ -114,7 +114,44 @@ - + + com.github.spotbugs + spotbugs-maven-plugin + 4.7.2.1 + + spotbugs-exclude.xml + + + + + com.github.spotbugs + spotbugs + 4.7.3 + + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.20.0 + + + + check + cpd-check + + + + + + + /pmd-rules.xml + + + + + + @@ -148,5 +185,15 @@ 1.10.19 test - + + + + + + org.apache.maven.plugins + maven-jxr-plugin + 3.3.0 + + + diff --git a/java/rocksjni/export_import_files_metadatajni.cc b/java/rocksjni/export_import_files_metadatajni.cc deleted file mode 100644 index a1de61933..000000000 --- a/java/rocksjni/export_import_files_metadatajni.cc +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -#include "include/org_rocksdb_ExportImportFilesMetaData.h" -#include "include/org_rocksdb_LiveFileMetaData.h" -#include "rocksjni/portal.h" - -/* - * Class: org_rocksdb_ExportImportFilesMetaData - * Method: newExportImportFilesMetaDataHandle - * Signature: ([BI[J)J - */ -jlong Java_org_rocksdb_ExportImportFilesMetaData_newExportImportFilesMetaDataHandle( - JNIEnv* env, jobject, jbyteArray j_db_comparator_name, - jint j_db_comparator_name_len, jlongArray j_live_file_meta_data_array) { - std::string db_comparator_name; - jboolean has_exception = JNI_FALSE; - - if (j_db_comparator_name_len > 0) { - db_comparator_name = ROCKSDB_NAMESPACE::JniUtil::byteString( - env, j_db_comparator_name, j_db_comparator_name_len, - [](const char* str, const size_t len) { return std::string(str, len); }, - &has_exception); - if (has_exception == JNI_TRUE) { - // exception occurred - return 0; - } - } - - std::vector live_file_metas; - jlong* ptr_live_file_meta_data_array = - env->GetLongArrayElements(j_live_file_meta_data_array, nullptr); - if (ptr_live_file_meta_data_array == nullptr) { - // exception thrown: OutOfMemoryError - return 0; - } - const jsize array_size = env->GetArrayLength(j_live_file_meta_data_array); - for (jsize i = 0; i < array_size; ++i) { - auto* ptr_level_file_meta = - reinterpret_cast( - ptr_live_file_meta_data_array[i]); - live_file_metas.push_back(*ptr_level_file_meta); - } - - env->ReleaseLongArrayElements(j_live_file_meta_data_array, - ptr_live_file_meta_data_array, JNI_ABORT); - auto* export_import_files_meta_data = - new ROCKSDB_NAMESPACE::ExportImportFilesMetaData; - export_import_files_meta_data->db_comparator_name = db_comparator_name; - export_import_files_meta_data->files = live_file_metas; - return GET_CPLUSPLUS_POINTER(export_import_files_meta_data); -} - -/* - * Class: org_rocksdb_LiveFileMetaData - * Method: newLiveFileMetaDataHandle - * Signature: ([BIILjava/lang/String;Ljava/lang/String;JJJ[BI[BIJZJJ)J - */ -jlong Java_org_rocksdb_LiveFileMetaData_newLiveFileMetaDataHandle( - JNIEnv* env, jobject, jbyteArray j_column_family_name, - jint j_column_family_name_len, jint j_level, jstring j_file_name, - jstring j_path, jlong j_size, jlong j_smallest_seqno, jlong j_largest_seqno, - jbyteArray j_smallest_key, jint j_smallest_key_len, - jbyteArray j_largest_key, jint j_largest_key_len, jlong j_num_read_sampled, - jboolean j_being_compacted, jlong j_num_entries, jlong j_num_deletions) { - std::string column_family_name; - jboolean has_exception = JNI_FALSE; - - if (j_column_family_name_len > 0) { - column_family_name = ROCKSDB_NAMESPACE::JniUtil::byteString( - env, j_column_family_name, j_column_family_name_len, - [](const char* str, const size_t len) { return std::string(str, len); }, - &has_exception); - if (has_exception == JNI_TRUE) { - // exception occurred - return 0; - } - } - - const char* file_name = env->GetStringUTFChars(j_file_name, nullptr); - if (file_name == nullptr) { - // exception thrown: OutOfMemoryError - return 0; - } - - const char* path = env->GetStringUTFChars(j_path, nullptr); - if (path == nullptr) { - // exception thrown: OutOfMemoryError - return 0; - } - - std::string smallest_key; - if (j_smallest_key_len > 0) { - smallest_key = ROCKSDB_NAMESPACE::JniUtil::byteString( - env, j_smallest_key, j_smallest_key_len, - [](const char* str, const size_t len) { return std::string(str, len); }, - &has_exception); - if (has_exception == JNI_TRUE) { - // exception occurred - return 0; - } - } - - std::string largest_key; - if (j_largest_key_len > 0) { - largest_key = ROCKSDB_NAMESPACE::JniUtil::byteString( - env, j_largest_key, j_largest_key_len, - [](const char* str, const size_t len) { return std::string(str, len); }, - &has_exception); - if (has_exception == JNI_TRUE) { - // exception occurred - return 0; - } - } - auto* live_file_meta = new ROCKSDB_NAMESPACE::LiveFileMetaData; - live_file_meta->column_family_name = column_family_name; - live_file_meta->level = static_cast(j_level); - live_file_meta->db_path = path; - live_file_meta->name = file_name; - live_file_meta->size = j_size; - live_file_meta->smallest_seqno = j_smallest_seqno; - live_file_meta->largest_seqno = j_largest_seqno; - live_file_meta->smallestkey = smallest_key; - live_file_meta->largestkey = largest_key; - live_file_meta->num_reads_sampled = j_num_read_sampled; - live_file_meta->being_compacted = j_being_compacted; - live_file_meta->num_entries = j_num_entries; - live_file_meta->num_deletions = j_num_deletions; - return GET_CPLUSPLUS_POINTER(live_file_meta); -} diff --git a/java/samples/src/main/java/OptimisticTransactionSample.java b/java/samples/src/main/java/OptimisticTransactionSample.java index 7e7a22e94..63c09d23b 100644 --- a/java/samples/src/main/java/OptimisticTransactionSample.java +++ b/java/samples/src/main/java/OptimisticTransactionSample.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -import org.rocksdb.*; +import org.forstdb.*; import static java.nio.charset.StandardCharsets.UTF_8; diff --git a/java/samples/src/main/java/RocksDBColumnFamilySample.java b/java/samples/src/main/java/RocksDBColumnFamilySample.java index 72f5731a1..8aaa8a793 100644 --- a/java/samples/src/main/java/RocksDBColumnFamilySample.java +++ b/java/samples/src/main/java/RocksDBColumnFamilySample.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -import org.rocksdb.*; +import org.forstdb.*; import java.util.ArrayList; import java.util.List; diff --git a/java/samples/src/main/java/RocksDBSample.java b/java/samples/src/main/java/RocksDBSample.java index 8ab9b2de3..3f8960093 100644 --- a/java/samples/src/main/java/RocksDBSample.java +++ b/java/samples/src/main/java/RocksDBSample.java @@ -9,8 +9,8 @@ import java.util.Map; import java.util.ArrayList; -import org.rocksdb.*; -import org.rocksdb.util.SizeUnit; +import org.forstdb.*; +import org.forstdb.util.SizeUnit; public class RocksDBSample { static { diff --git a/java/samples/src/main/java/TransactionSample.java b/java/samples/src/main/java/TransactionSample.java index b88a68f12..81102d972 100644 --- a/java/samples/src/main/java/TransactionSample.java +++ b/java/samples/src/main/java/TransactionSample.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -import org.rocksdb.*; +import org.forstdb.*; import static java.nio.charset.StandardCharsets.UTF_8; diff --git a/java/spotbugs-exclude.xml b/java/spotbugs-exclude.xml new file mode 100644 index 000000000..0c8d44929 --- /dev/null +++ b/java/spotbugs-exclude.xml @@ -0,0 +1,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/src/main/java/org/rocksdb/AbstractCompactionFilter.java b/java/src/main/java/org/forstdb/AbstractCompactionFilter.java similarity index 98% rename from java/src/main/java/org/rocksdb/AbstractCompactionFilter.java rename to java/src/main/java/org/forstdb/AbstractCompactionFilter.java index fd7eef4d4..fc401252e 100644 --- a/java/src/main/java/org/rocksdb/AbstractCompactionFilter.java +++ b/java/src/main/java/org/forstdb/AbstractCompactionFilter.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * A CompactionFilter allows an application to modify/delete a key-value at diff --git a/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java b/java/src/main/java/org/forstdb/AbstractCompactionFilterFactory.java similarity index 92% rename from java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java rename to java/src/main/java/org/forstdb/AbstractCompactionFilterFactory.java index 4bb985a34..0fbfcb839 100644 --- a/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java +++ b/java/src/main/java/org/forstdb/AbstractCompactionFilterFactory.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Each compaction will create a new {@link AbstractCompactionFilter} @@ -31,8 +31,9 @@ protected long initializeNative(final long... nativeParameterHandles) { * * @return native handle of the CompactionFilter */ - private long createCompactionFilter(final boolean fullCompaction, - final boolean manualCompaction) { + @SuppressWarnings({"PMD.UnusedPrivateMethod", "PMD.CloseResource"}) + private long createCompactionFilter( + final boolean fullCompaction, final boolean manualCompaction) { final T filter = createCompactionFilter( new AbstractCompactionFilter.Context(fullCompaction, manualCompaction)); diff --git a/java/src/main/java/org/rocksdb/AbstractComparator.java b/java/src/main/java/org/forstdb/AbstractComparator.java similarity index 99% rename from java/src/main/java/org/rocksdb/AbstractComparator.java rename to java/src/main/java/org/forstdb/AbstractComparator.java index 83e0f0676..f66a663ce 100644 --- a/java/src/main/java/org/rocksdb/AbstractComparator.java +++ b/java/src/main/java/org/forstdb/AbstractComparator.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; diff --git a/java/src/main/java/org/rocksdb/AbstractComparatorJniBridge.java b/java/src/main/java/org/forstdb/AbstractComparatorJniBridge.java similarity index 94% rename from java/src/main/java/org/rocksdb/AbstractComparatorJniBridge.java rename to java/src/main/java/org/forstdb/AbstractComparatorJniBridge.java index 2d1bf702b..c7e1fa1df 100644 --- a/java/src/main/java/org/rocksdb/AbstractComparatorJniBridge.java +++ b/java/src/main/java/org/forstdb/AbstractComparatorJniBridge.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; @@ -15,7 +15,7 @@ * * Placing these bridge methods in this * class keeps the API of the - * {@link org.rocksdb.AbstractComparator} clean. + * {@link org.forstdb.AbstractComparator} clean. */ class AbstractComparatorJniBridge { /** @@ -37,6 +37,7 @@ class AbstractComparatorJniBridge { * * @return the result of the comparison */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private static int compareInternal(final AbstractComparator comparator, final ByteBuffer a, final int aLen, final ByteBuffer b, final int bLen) { if (aLen != -1) { @@ -80,6 +81,7 @@ private static int compareInternal(final AbstractComparator comparator, final By * @return either {@code startLen} if the start key is unchanged, otherwise * the new length of the start key */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private static int findShortestSeparatorInternal(final AbstractComparator comparator, final ByteBuffer start, final int startLen, final ByteBuffer limit, final int limitLen) { if (startLen != -1) { @@ -108,6 +110,7 @@ private static int findShortestSeparatorInternal(final AbstractComparator compar * * @return either keyLen if the key is unchanged, otherwise the new length of the key */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private static int findShortSuccessorInternal( final AbstractComparator comparator, final ByteBuffer key, final int keyLen) { if (keyLen != -1) { diff --git a/java/src/main/java/org/rocksdb/AbstractEventListener.java b/java/src/main/java/org/forstdb/AbstractEventListener.java similarity index 91% rename from java/src/main/java/org/rocksdb/AbstractEventListener.java rename to java/src/main/java/org/forstdb/AbstractEventListener.java index d640d3423..d03a599be 100644 --- a/java/src/main/java/org/rocksdb/AbstractEventListener.java +++ b/java/src/main/java/org/forstdb/AbstractEventListener.java @@ -3,13 +3,14 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; -import static org.rocksdb.AbstractEventListener.EnabledEventCallback.*; +import static org.forstdb.AbstractEventListener.EnabledEventCallback.*; /** * Base class for Event Listeners. */ +@SuppressWarnings("PMD.AvoidDuplicateLiterals") public abstract class AbstractEventListener extends RocksCallbackObject implements EventListener { public enum EnabledEventCallback { ON_FLUSH_COMPLETED((byte) 0x0), @@ -58,7 +59,7 @@ byte getValue() { * @throws IllegalArgumentException if the value is unknown. */ static EnabledEventCallback fromValue(final byte value) { - for (final EnabledEventCallback enabledEventCallback : EnabledEventCallback.values()) { + for (final EnabledEventCallback enabledEventCallback : values()) { if (enabledEventCallback.value == value) { return enabledEventCallback; } @@ -124,8 +125,9 @@ public void onFlushCompleted(final RocksDB db, final FlushJobInfo flushJobInfo) * @param dbHandle native handle of the database * @param flushJobInfo the flush job info */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private void onFlushCompletedProxy(final long dbHandle, final FlushJobInfo flushJobInfo) { - final RocksDB db = new RocksDB(dbHandle); + final RocksDB db = new RocksDB(dbHandle); // NOPMD - CloseResource db.disOwnNativeHandle(); // we don't own this! onFlushCompleted(db, flushJobInfo); } @@ -142,8 +144,9 @@ public void onFlushBegin(final RocksDB db, final FlushJobInfo flushJobInfo) { * @param dbHandle native handle of the database * @param flushJobInfo the flush job info */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private void onFlushBeginProxy(final long dbHandle, final FlushJobInfo flushJobInfo) { - final RocksDB db = new RocksDB(dbHandle); + final RocksDB db = new RocksDB(dbHandle); // NOPMD - CloseResource db.disOwnNativeHandle(); // we don't own this! onFlushBegin(db, flushJobInfo); } @@ -165,9 +168,10 @@ public void onCompactionBegin(final RocksDB db, final CompactionJobInfo compacti * @param dbHandle native handle of the database * @param compactionJobInfo the flush job info */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private void onCompactionBeginProxy( final long dbHandle, final CompactionJobInfo compactionJobInfo) { - final RocksDB db = new RocksDB(dbHandle); + final RocksDB db = new RocksDB(dbHandle); // NOPMD - CloseResource db.disOwnNativeHandle(); // we don't own this! onCompactionBegin(db, compactionJobInfo); } @@ -184,9 +188,10 @@ public void onCompactionCompleted(final RocksDB db, final CompactionJobInfo comp * @param dbHandle native handle of the database * @param compactionJobInfo the flush job info */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private void onCompactionCompletedProxy( final long dbHandle, final CompactionJobInfo compactionJobInfo) { - final RocksDB db = new RocksDB(dbHandle); + final RocksDB db = new RocksDB(dbHandle); // NOPMD - CloseResource db.disOwnNativeHandle(); // we don't own this! onCompactionCompleted(db, compactionJobInfo); } @@ -225,9 +230,10 @@ public void onExternalFileIngested( * @param dbHandle native handle of the database * @param externalFileIngestionInfo the flush job info */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private void onExternalFileIngestedProxy( final long dbHandle, final ExternalFileIngestionInfo externalFileIngestionInfo) { - final RocksDB db = new RocksDB(dbHandle); + final RocksDB db = new RocksDB(dbHandle); // NOPMD - CloseResource db.disOwnNativeHandle(); // we don't own this! onExternalFileIngested(db, externalFileIngestionInfo); } @@ -245,6 +251,7 @@ public void onBackgroundError( * @param reasonByte byte value representing error reason * @param backgroundError status with error code */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private void onBackgroundErrorProxy(final byte reasonByte, final Status backgroundError) { onBackgroundError(BackgroundErrorReason.fromValue(reasonByte), backgroundError); } @@ -307,6 +314,7 @@ public boolean onErrorRecoveryBegin( * @param reasonByte byte value representing error reason * @param backgroundError status with error code */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private boolean onErrorRecoveryBeginProxy(final byte reasonByte, final Status backgroundError) { return onErrorRecoveryBegin(BackgroundErrorReason.fromValue(reasonByte), backgroundError); } diff --git a/java/src/main/java/org/rocksdb/AbstractImmutableNativeReference.java b/java/src/main/java/org/forstdb/AbstractImmutableNativeReference.java similarity index 99% rename from java/src/main/java/org/rocksdb/AbstractImmutableNativeReference.java rename to java/src/main/java/org/forstdb/AbstractImmutableNativeReference.java index 173d63e90..7e667bbea 100644 --- a/java/src/main/java/org/rocksdb/AbstractImmutableNativeReference.java +++ b/java/src/main/java/org/forstdb/AbstractImmutableNativeReference.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/java/src/main/java/org/rocksdb/AbstractMutableOptions.java b/java/src/main/java/org/forstdb/AbstractMutableOptions.java similarity index 90% rename from java/src/main/java/org/rocksdb/AbstractMutableOptions.java rename to java/src/main/java/org/forstdb/AbstractMutableOptions.java index 1a6251bd4..802ca7c81 100644 --- a/java/src/main/java/org/rocksdb/AbstractMutableOptions.java +++ b/java/src/main/java/org/forstdb/AbstractMutableOptions.java @@ -1,14 +1,20 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import java.util.*; -public abstract class AbstractMutableOptions { - +/** + * This class is not strictly abstract in Java language terms, so we do not declare it as such. + * The name remains {@code AbstractMutableOptions} to reflect the underlying C++ name. + * The constructor is protected, so it will always be used as a base class. + */ +public class AbstractMutableOptions { protected static final String KEY_VALUE_PAIR_SEPARATOR = ";"; protected static final char KEY_VALUE_SEPARATOR = '='; static final String INT_ARRAY_INT_SEPARATOR = ":"; + private static final String HAS_NOT_BEEN_SET = " has not been set"; + protected final String[] keys; private final String[] values; @@ -18,15 +24,18 @@ public abstract class AbstractMutableOptions { * @param keys the keys * @param values the values */ + @SuppressWarnings("PMD.ArrayIsStoredDirectly") protected AbstractMutableOptions(final String[] keys, final String[] values) { this.keys = keys; this.values = values; } + @SuppressWarnings("PMD.MethodReturnsInternalArray") String[] getKeys() { return keys; } + @SuppressWarnings("PMD.MethodReturnsInternalArray") String[] getValues() { return values; } @@ -106,7 +115,7 @@ protected double getDouble(final K key) throws NoSuchElementException, NumberFormatException { final MutableOptionValue value = options.get(key); if(value == null) { - throw new NoSuchElementException(key.name() + " has not been set"); + throw new NoSuchElementException(key.name() + HAS_NOT_BEEN_SET); } return value.asDouble(); } @@ -125,7 +134,7 @@ protected long getLong(final K key) throws NoSuchElementException, NumberFormatException { final MutableOptionValue value = options.get(key); if(value == null) { - throw new NoSuchElementException(key.name() + " has not been set"); + throw new NoSuchElementException(key.name() + HAS_NOT_BEEN_SET); } return value.asLong(); } @@ -144,7 +153,7 @@ protected int getInt(final K key) throws NoSuchElementException, NumberFormatException { final MutableOptionValue value = options.get(key); if(value == null) { - throw new NoSuchElementException(key.name() + " has not been set"); + throw new NoSuchElementException(key.name() + HAS_NOT_BEEN_SET); } return value.asInt(); } @@ -163,7 +172,7 @@ protected boolean getBoolean(final K key) throws NoSuchElementException, NumberFormatException { final MutableOptionValue value = options.get(key); if(value == null) { - throw new NoSuchElementException(key.name() + " has not been set"); + throw new NoSuchElementException(key.name() + HAS_NOT_BEEN_SET); } return value.asBoolean(); } @@ -182,7 +191,7 @@ protected int[] getIntArray(final K key) throws NoSuchElementException, NumberFormatException { final MutableOptionValue value = options.get(key); if(value == null) { - throw new NoSuchElementException(key.name() + " has not been set"); + throw new NoSuchElementException(key.name() + HAS_NOT_BEEN_SET); } return value.asIntArray(); } @@ -202,7 +211,7 @@ protected > N getEnum(final K key) throws NoSuchElementException, NumberFormatException { final MutableOptionValue value = options.get(key); if (value == null) { - throw new NoSuchElementException(key.name() + " has not been set"); + throw new NoSuchElementException(key.name() + HAS_NOT_BEEN_SET); } if (!(value instanceof MutableOptionValue.MutableOptionEnumValue)) { @@ -225,7 +234,7 @@ private long parseAsLong(final String value) { } catch (final NumberFormatException nfe) { final double doubleValue = Double.parseDouble(value); if (doubleValue != Math.round(doubleValue)) - throw new IllegalArgumentException("Unable to parse or round " + value + " to long"); + throw new IllegalArgumentException("Unable to parse or round " + value + " to long", nfe); return Math.round(doubleValue); } } @@ -243,7 +252,7 @@ private int parseAsInt(final String value) { } catch (final NumberFormatException nfe) { final double doubleValue = Double.parseDouble(value); if (doubleValue != Math.round(doubleValue)) - throw new IllegalArgumentException("Unable to parse or round " + value + " to int"); + throw new IllegalArgumentException("Unable to parse or round " + value + " to int", nfe); return (int) Math.round(doubleValue); } } @@ -287,6 +296,7 @@ protected U fromParsed(final List options, final boolean ign * @return the same object, after adding options * @throws IllegalArgumentException if the key is unknown, or a value has the wrong type/form */ + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") private U fromOptionString(final OptionString.Entry option, final boolean ignoreUnknown) throws IllegalArgumentException { Objects.requireNonNull(option.key); @@ -340,12 +350,12 @@ private U fromOptionString(final OptionString.Entry option, final boolean ignore case ENUM: final String optionName = key.name(); - if (optionName.equals("prepopulate_blob_cache")) { + if ("prepopulate_blob_cache".equals(optionName)) { final PrepopulateBlobCache prepopulateBlobCache = PrepopulateBlobCache.getFromInternal(valueStr); return setEnum(key, prepopulateBlobCache); - } else if (optionName.equals("compression") - || optionName.equals("blob_compression_type")) { + } else if ("compression".equals(optionName) + || "blob_compression_type".equals(optionName)) { final CompressionType compressionType = CompressionType.getFromInternal(valueStr); return setEnum(key, compressionType); } else { diff --git a/java/src/main/java/org/rocksdb/AbstractNativeReference.java b/java/src/main/java/org/forstdb/AbstractNativeReference.java similarity index 98% rename from java/src/main/java/org/rocksdb/AbstractNativeReference.java rename to java/src/main/java/org/forstdb/AbstractNativeReference.java index 1ce54fcba..b0cc585d1 100644 --- a/java/src/main/java/org/rocksdb/AbstractNativeReference.java +++ b/java/src/main/java/org/forstdb/AbstractNativeReference.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * AbstractNativeReference is the base-class of all RocksDB classes that have diff --git a/java/src/main/java/org/rocksdb/AbstractRocksIterator.java b/java/src/main/java/org/forstdb/AbstractRocksIterator.java similarity index 98% rename from java/src/main/java/org/rocksdb/AbstractRocksIterator.java rename to java/src/main/java/org/forstdb/AbstractRocksIterator.java index 1aade1b89..a68b63157 100644 --- a/java/src/main/java/org/rocksdb/AbstractRocksIterator.java +++ b/java/src/main/java/org/forstdb/AbstractRocksIterator.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; @@ -19,7 +19,7 @@ * @param

The type of the Parent Object from which the Rocks Iterator was * created. This is used by disposeInternal to avoid double-free * issues with the underlying C++ object. - * @see org.rocksdb.RocksObject + * @see org.forstdb.RocksObject */ public abstract class AbstractRocksIterator

extends RocksObject implements RocksIteratorInterface { diff --git a/java/src/main/java/org/rocksdb/AbstractSlice.java b/java/src/main/java/org/forstdb/AbstractSlice.java similarity index 88% rename from java/src/main/java/org/rocksdb/AbstractSlice.java rename to java/src/main/java/org/forstdb/AbstractSlice.java index 0681b6758..afc6b3e70 100644 --- a/java/src/main/java/org/rocksdb/AbstractSlice.java +++ b/java/src/main/java/org/forstdb/AbstractSlice.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Slices are used by RocksDB to provide @@ -11,18 +11,18 @@ *

* This class is package private, implementers * should extend either of the public abstract classes: - * @see org.rocksdb.Slice - * @see org.rocksdb.DirectSlice + * @see org.forstdb.Slice + * @see org.forstdb.DirectSlice * * Regards the lifecycle of Java Slices in RocksDB: * At present when you configure a Comparator from Java, it creates an * instance of a C++ BaseComparatorJniCallback subclass and * passes that to RocksDB as the comparator. That subclass of * BaseComparatorJniCallback creates the Java - * @see org.rocksdb.AbstractSlice subclass Objects. When you dispose - * the Java @see org.rocksdb.AbstractComparator subclass, it disposes the + * @see org.forstdb.AbstractSlice subclass Objects. When you dispose + * the Java @see org.forstdb.AbstractComparator subclass, it disposes the * C++ BaseComparatorJniCallback subclass, which in turn destroys the - * Java @see org.rocksdb.AbstractSlice subclass Objects. + * Java @see org.forstdb.AbstractSlice subclass Objects. */ public abstract class AbstractSlice extends RocksMutableObject { @@ -39,7 +39,7 @@ protected AbstractSlice(final long nativeHandle) { * * @return The slice data. Note, the type of access is * determined by the subclass - * @see org.rocksdb.AbstractSlice#data0(long) + * @see org.forstdb.AbstractSlice#data0(long) */ public T data() { return data0(getNativeHandle()); @@ -119,14 +119,16 @@ public String toString() { */ public int compare(final AbstractSlice other) { assert (other != null); - if(!isOwningHandle()) { - return other.isOwningHandle() ? -1 : 0; + if (isOwningHandle() && other.isOwningHandle()) { + return compare0(getNativeHandle(), other.getNativeHandle()); + } + if (!isOwningHandle() && !other.isOwningHandle()) { + return 0; + } + if (isOwningHandle()) { + return 1; } else { - if(!other.isOwningHandle()) { - return 1; - } else { - return compare0(getNativeHandle(), other.getNativeHandle()); - } + return -1; } } diff --git a/java/src/main/java/org/rocksdb/AbstractTableFilter.java b/java/src/main/java/org/forstdb/AbstractTableFilter.java similarity index 95% rename from java/src/main/java/org/rocksdb/AbstractTableFilter.java rename to java/src/main/java/org/forstdb/AbstractTableFilter.java index c696c3e13..d83a9839f 100644 --- a/java/src/main/java/org/rocksdb/AbstractTableFilter.java +++ b/java/src/main/java/org/forstdb/AbstractTableFilter.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** * Base class for Table Filters. diff --git a/java/src/main/java/org/rocksdb/AbstractTraceWriter.java b/java/src/main/java/org/forstdb/AbstractTraceWriter.java similarity index 95% rename from java/src/main/java/org/rocksdb/AbstractTraceWriter.java rename to java/src/main/java/org/forstdb/AbstractTraceWriter.java index 13edfbd84..aee94cf94 100644 --- a/java/src/main/java/org/rocksdb/AbstractTraceWriter.java +++ b/java/src/main/java/org/forstdb/AbstractTraceWriter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Base class for TraceWriters. @@ -25,6 +25,7 @@ protected long initializeNative(final long... nativeParameterHandles) { * {@link Status.Code#getValue()} and the second byte is the * {@link Status.SubCode#getValue()}. */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private short writeProxy(final long sliceHandle) { try { write(new Slice(sliceHandle)); @@ -41,6 +42,7 @@ private short writeProxy(final long sliceHandle) { * {@link Status.Code#getValue()} and the second byte is the * {@link Status.SubCode#getValue()}. */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private short closeWriterProxy() { try { closeWriter(); diff --git a/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java b/java/src/main/java/org/forstdb/AbstractTransactionNotifier.java similarity index 98% rename from java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java rename to java/src/main/java/org/forstdb/AbstractTransactionNotifier.java index b117e5cc2..d0c98eab2 100644 --- a/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java +++ b/java/src/main/java/org/forstdb/AbstractTransactionNotifier.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Provides notification to the caller of SetSnapshotOnNextOperation when diff --git a/java/src/main/java/org/rocksdb/AbstractWalFilter.java b/java/src/main/java/org/forstdb/AbstractWalFilter.java similarity index 88% rename from java/src/main/java/org/rocksdb/AbstractWalFilter.java rename to java/src/main/java/org/forstdb/AbstractWalFilter.java index fc77eab8e..2a1cb9095 100644 --- a/java/src/main/java/org/rocksdb/AbstractWalFilter.java +++ b/java/src/main/java/org/forstdb/AbstractWalFilter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Base class for WAL Filters. @@ -30,9 +30,9 @@ protected long initializeNative(final long... nativeParameterHandles) { * {@link WalFilter.LogRecordFoundResult#walProcessingOption} * {@link WalFilter.LogRecordFoundResult#batchChanged}. */ - private short logRecordFoundProxy(final long logNumber, - final String logFileName, final long batchHandle, - final long newBatchHandle) { + @SuppressWarnings("PMD.UnusedPrivateMethod") + private short logRecordFoundProxy(final long logNumber, final String logFileName, + final long batchHandle, final long newBatchHandle) { final LogRecordFoundResult logRecordFoundResult = logRecordFound( logNumber, logFileName, new WriteBatch(batchHandle), new WriteBatch(newBatchHandle)); diff --git a/java/src/main/java/org/rocksdb/AbstractWriteBatch.java b/java/src/main/java/org/forstdb/AbstractWriteBatch.java similarity index 99% rename from java/src/main/java/org/rocksdb/AbstractWriteBatch.java rename to java/src/main/java/org/forstdb/AbstractWriteBatch.java index 41d967f53..2bb2ec324 100644 --- a/java/src/main/java/org/rocksdb/AbstractWriteBatch.java +++ b/java/src/main/java/org/forstdb/AbstractWriteBatch.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; diff --git a/java/src/main/java/org/rocksdb/AccessHint.java b/java/src/main/java/org/forstdb/AccessHint.java similarity index 97% rename from java/src/main/java/org/rocksdb/AccessHint.java rename to java/src/main/java/org/forstdb/AccessHint.java index 877c4ab39..a70968f7c 100644 --- a/java/src/main/java/org/rocksdb/AccessHint.java +++ b/java/src/main/java/org/forstdb/AccessHint.java @@ -3,11 +3,12 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * File access pattern once a compaction has started */ +@Deprecated public enum AccessHint { NONE((byte)0x0), NORMAL((byte)0x1), diff --git a/java/src/main/java/org/rocksdb/AdvancedColumnFamilyOptionsInterface.java b/java/src/main/java/org/forstdb/AdvancedColumnFamilyOptionsInterface.java similarity index 98% rename from java/src/main/java/org/rocksdb/AdvancedColumnFamilyOptionsInterface.java rename to java/src/main/java/org/forstdb/AdvancedColumnFamilyOptionsInterface.java index d1d1123dd..27c5f9f47 100644 --- a/java/src/main/java/org/rocksdb/AdvancedColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/forstdb/AdvancedColumnFamilyOptionsInterface.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.List; @@ -192,7 +192,7 @@ T setInplaceUpdateSupport( *

Default: empty

* * @param compressionLevels list of - * {@link org.rocksdb.CompressionType} instances. + * {@link org.forstdb.CompressionType} instances. * * @return the reference to the current options. */ @@ -200,12 +200,12 @@ T setCompressionPerLevel( List compressionLevels); /** - *

Return the currently set {@link org.rocksdb.CompressionType} + *

Return the currently set {@link org.forstdb.CompressionType} * per instances.

* *

See: {@link #setCompressionPerLevel(java.util.List)}

* - * @return list of {@link org.rocksdb.CompressionType} + * @return list of {@link org.forstdb.CompressionType} * instances. */ List compressionPerLevel(); diff --git a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java b/java/src/main/java/org/forstdb/AdvancedMutableColumnFamilyOptionsInterface.java similarity index 99% rename from java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java rename to java/src/main/java/org/forstdb/AdvancedMutableColumnFamilyOptionsInterface.java index c8fc84173..1b6717e19 100644 --- a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/forstdb/AdvancedMutableColumnFamilyOptionsInterface.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Advanced Column Family Options which are mutable diff --git a/java/src/main/java/org/rocksdb/BackgroundErrorReason.java b/java/src/main/java/org/forstdb/BackgroundErrorReason.java similarity index 98% rename from java/src/main/java/org/rocksdb/BackgroundErrorReason.java rename to java/src/main/java/org/forstdb/BackgroundErrorReason.java index eec593d35..11d6431af 100644 --- a/java/src/main/java/org/rocksdb/BackgroundErrorReason.java +++ b/java/src/main/java/org/forstdb/BackgroundErrorReason.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum BackgroundErrorReason { FLUSH((byte) 0x0), diff --git a/java/src/main/java/org/rocksdb/BackupEngine.java b/java/src/main/java/org/forstdb/BackupEngine.java similarity index 99% rename from java/src/main/java/org/rocksdb/BackupEngine.java rename to java/src/main/java/org/forstdb/BackupEngine.java index 3ab220683..7f4298d1d 100644 --- a/java/src/main/java/org/rocksdb/BackupEngine.java +++ b/java/src/main/java/org/forstdb/BackupEngine.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.List; diff --git a/java/src/main/java/org/rocksdb/BackupEngineOptions.java b/java/src/main/java/org/forstdb/BackupEngineOptions.java similarity index 96% rename from java/src/main/java/org/rocksdb/BackupEngineOptions.java rename to java/src/main/java/org/forstdb/BackupEngineOptions.java index 2a358faac..8f5bf6f0f 100644 --- a/java/src/main/java/org/rocksdb/BackupEngineOptions.java +++ b/java/src/main/java/org/forstdb/BackupEngineOptions.java @@ -3,18 +3,18 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.io.File; /** *

BackupEngineOptions controls the behavior of a - * {@link org.rocksdb.BackupEngine}. + * {@link org.forstdb.BackupEngine}. *

*

Note that dispose() must be called before an Options instance * become out-of-scope to release the allocated memory in c++.

* - * @see org.rocksdb.BackupEngine + * @see org.forstdb.BackupEngine */ public class BackupEngineOptions extends RocksObject { private Env backupEnv = null; @@ -228,10 +228,9 @@ public boolean backupLogFiles() { * * @return instance of current BackupEngineOptions. */ - public BackupEngineOptions setBackupRateLimit(long backupRateLimit) { + public BackupEngineOptions setBackupRateLimit(final long backupRateLimit) { assert(isOwningHandle()); - backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit; - setBackupRateLimit(nativeHandle_, backupRateLimit); + setBackupRateLimit(nativeHandle_, (backupRateLimit <= 0) ? 0 : backupRateLimit); return this; } @@ -286,10 +285,9 @@ public RateLimiter backupRateLimiter() { * * @return instance of current BackupEngineOptions. */ - public BackupEngineOptions setRestoreRateLimit(long restoreRateLimit) { + public BackupEngineOptions setRestoreRateLimit(final long restoreRateLimit) { assert(isOwningHandle()); - restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit; - setRestoreRateLimit(nativeHandle_, restoreRateLimit); + setRestoreRateLimit(nativeHandle_, (restoreRateLimit <= 0) ? 0 : restoreRateLimit); return this; } diff --git a/java/src/main/java/org/rocksdb/BackupInfo.java b/java/src/main/java/org/forstdb/BackupInfo.java similarity index 93% rename from java/src/main/java/org/rocksdb/BackupInfo.java rename to java/src/main/java/org/forstdb/BackupInfo.java index 9581b098f..24179123c 100644 --- a/java/src/main/java/org/rocksdb/BackupInfo.java +++ b/java/src/main/java/org/forstdb/BackupInfo.java @@ -2,17 +2,17 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Instances of this class describe a Backup made by - * {@link org.rocksdb.BackupEngine}. + * {@link org.forstdb.BackupEngine}. */ public class BackupInfo { /** * Package private constructor used to create instances - * of BackupInfo by {@link org.rocksdb.BackupEngine} + * of BackupInfo by {@link org.forstdb.BackupEngine} * * @param backupId id of backup * @param timestamp timestamp of backup diff --git a/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java b/java/src/main/java/org/forstdb/BlockBasedTableConfig.java similarity index 89% rename from java/src/main/java/org/rocksdb/BlockBasedTableConfig.java rename to java/src/main/java/org/forstdb/BlockBasedTableConfig.java index 70dee3dd9..9723eb3d9 100644 --- a/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java +++ b/java/src/main/java/org/forstdb/BlockBasedTableConfig.java @@ -2,16 +2,16 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** - * The config for plain table sst format. + * The config for block based table sst format. *

* BlockBasedTable is a RocksDB's default SST file format. */ // TODO(AR) should be renamed BlockBasedTableOptions public class BlockBasedTableConfig extends TableFormatConfig { - + @SuppressWarnings("PMD.NullAssignment") public BlockBasedTableConfig() { //TODO(AR) flushBlockPolicyFactory cacheIndexAndFilterBlocks = false; @@ -21,7 +21,7 @@ public BlockBasedTableConfig() { indexType = IndexType.kBinarySearch; dataBlockIndexType = DataBlockIndexType.kDataBlockBinarySearch; dataBlockHashTableUtilRatio = 0.75; - checksumType = ChecksumType.kCRC32c; + checksumType = ChecksumType.kXXH3; noBlockCache = false; blockCache = null; persistentCache = null; @@ -47,6 +47,55 @@ public BlockBasedTableConfig() { blockCacheNumShardBits = 0; } + /** + * Constructor for use by C++ via JNI + */ + private BlockBasedTableConfig(final boolean cacheIndexAndFilterBlocks, + final boolean cacheIndexAndFilterBlocksWithHighPriority, + final boolean pinL0FilterAndIndexBlocksInCache, final boolean pinTopLevelIndexAndFilter, + final byte indexType, final byte dataBlockIndexType, final double dataBlockHashTableUtilRatio, + final byte checksumType, final boolean noBlockCache, final long blockSize, + final int blockSizeDeviation, final int blockRestartInterval, + final int indexBlockRestartInterval, final long metadataBlockSize, + final boolean partitionFilters, final boolean optimizeFiltersForMemory, + final boolean useDeltaEncoding, final boolean wholeKeyFiltering, + final boolean verifyCompression, final int readAmpBytesPerBit, final int formatVersion, + final boolean enableIndexCompression, final boolean blockAlign, final byte indexShortening, + final byte filterPolicyType, final long filterPolicyHandle, + final double filterPolicyConfigValue) { + this.cacheIndexAndFilterBlocks = cacheIndexAndFilterBlocks; + this.cacheIndexAndFilterBlocksWithHighPriority = cacheIndexAndFilterBlocksWithHighPriority; + this.pinL0FilterAndIndexBlocksInCache = pinL0FilterAndIndexBlocksInCache; + this.pinTopLevelIndexAndFilter = pinTopLevelIndexAndFilter; + this.indexType = IndexType.values()[indexType]; + this.dataBlockIndexType = DataBlockIndexType.values()[dataBlockIndexType]; + this.dataBlockHashTableUtilRatio = dataBlockHashTableUtilRatio; + this.checksumType = ChecksumType.values()[checksumType]; + this.noBlockCache = noBlockCache; + this.blockSize = blockSize; + this.blockSizeDeviation = blockSizeDeviation; + this.blockRestartInterval = blockRestartInterval; + this.indexBlockRestartInterval = indexBlockRestartInterval; + this.metadataBlockSize = metadataBlockSize; + this.partitionFilters = partitionFilters; + this.optimizeFiltersForMemory = optimizeFiltersForMemory; + this.useDeltaEncoding = useDeltaEncoding; + this.wholeKeyFiltering = wholeKeyFiltering; + this.verifyCompression = verifyCompression; + this.readAmpBytesPerBit = readAmpBytesPerBit; + this.formatVersion = formatVersion; + this.enableIndexCompression = enableIndexCompression; + this.blockAlign = blockAlign; + this.indexShortening = IndexShorteningMode.values()[indexShortening]; + try (Filter filterPolicy = FilterPolicyType.values()[filterPolicyType].createFilter( + filterPolicyHandle, filterPolicyConfigValue)) { + if (filterPolicy != null) { + filterPolicy.disOwnNativeHandle(); + this.setFilterPolicy(filterPolicy); + } + } + } + /** * Indicating if we'd put index/filter blocks to the block cache. * If not specified, each "table reader" object will pre-load index/filter @@ -156,7 +205,7 @@ public IndexType indexType() { /** * Sets the index type to used with this table. * - * @param indexType {@link org.rocksdb.IndexType} value + * @param indexType {@link org.forstdb.IndexType} value * @return the reference to the current option. */ public BlockBasedTableConfig setIndexType( @@ -177,7 +226,7 @@ public DataBlockIndexType dataBlockIndexType() { /** * Sets the data block index type to used with this table. * - * @param dataBlockIndexType {@link org.rocksdb.DataBlockIndexType} value + * @param dataBlockIndexType {@link org.forstdb.DataBlockIndexType} value * @return the reference to the current option. */ public BlockBasedTableConfig setDataBlockIndexType( @@ -221,7 +270,7 @@ public ChecksumType checksumType() { /** * Sets * - * @param checksumType {@link org.rocksdb.ChecksumType} value. + * @param checksumType {@link org.forstdb.ChecksumType} value. * @return the reference to the current option. */ public BlockBasedTableConfig setChecksumType( @@ -258,13 +307,13 @@ public BlockBasedTableConfig setNoBlockCache(final boolean noBlockCache) { * Use the specified cache for blocks. * When not null this take precedence even if the user sets a block cache size. *

- * {@link org.rocksdb.Cache} should not be disposed before options instances + * {@link org.forstdb.Cache} should not be disposed before options instances * using this cache is disposed. *

- * {@link org.rocksdb.Cache} instance can be re-used in multiple options + * {@link org.forstdb.Cache} instance can be re-used in multiple options * instances. * - * @param blockCache {@link org.rocksdb.Cache} Cache java instance + * @param blockCache {@link org.forstdb.Cache} Cache java instance * (e.g. LRUCache). * * @return the reference to the current config. @@ -522,13 +571,13 @@ public Filter filterPolicy() { /** * Use the specified filter policy to reduce disk reads. *

- * {@link org.rocksdb.Filter} should not be closed before options instances + * {@link org.forstdb.Filter} should not be closed before options instances * using this filter are closed. *

- * {@link org.rocksdb.Filter} instance can be re-used in multiple options + * {@link org.forstdb.Filter} instance can be re-used in multiple options * instances. * - * @param filterPolicy {@link org.rocksdb.Filter} Filter Policy java instance. + * @param filterPolicy {@link org.forstdb.Filter} Filter Policy java instance. * @return the reference to the current config. */ public BlockBasedTableConfig setFilterPolicy( diff --git a/java/src/main/java/org/rocksdb/BloomFilter.java b/java/src/main/java/org/forstdb/BloomFilter.java similarity index 72% rename from java/src/main/java/org/rocksdb/BloomFilter.java rename to java/src/main/java/org/forstdb/BloomFilter.java index 0b4e93229..ea1ebdce5 100644 --- a/java/src/main/java/org/rocksdb/BloomFilter.java +++ b/java/src/main/java/org/forstdb/BloomFilter.java @@ -3,7 +3,9 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; + +import java.util.Objects; /** * Bloom filter policy that uses a bloom filter with approximately @@ -33,6 +35,9 @@ public BloomFilter() { this(DEFAULT_BITS_PER_KEY); } + // record this for comparison of filters. + private final double bitsPerKey; + /** * BloomFilter constructor * @@ -47,7 +52,17 @@ public BloomFilter() { * @param bitsPerKey number of bits to use */ public BloomFilter(final double bitsPerKey) { - super(createNewBloomFilter(bitsPerKey)); + this(createNewBloomFilter(bitsPerKey), bitsPerKey); + } + + /** + * + * @param nativeHandle handle to existing bloom filter at RocksDB C++ side + * @param bitsPerKey number of bits to use - recorded for comparison + */ + BloomFilter(final long nativeHandle, final double bitsPerKey) { + super(nativeHandle); + this.bitsPerKey = bitsPerKey; } /** @@ -65,9 +80,25 @@ public BloomFilter(final double bitsPerKey) { * @param bitsPerKey number of bits to use * @param IGNORED_useBlockBasedMode obsolete, ignored parameter */ + @SuppressWarnings("PMD.UnusedFormalParameter") public BloomFilter(final double bitsPerKey, final boolean IGNORED_useBlockBasedMode) { this(bitsPerKey); } + @SuppressWarnings("PMD.") + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + return bitsPerKey == ((BloomFilter) o).bitsPerKey; + } + + @Override + public int hashCode() { + return Objects.hash(bitsPerKey); + } + private static native long createNewBloomFilter(final double bitsKeyKey); } diff --git a/java/src/main/java/org/rocksdb/BuiltinComparator.java b/java/src/main/java/org/forstdb/BuiltinComparator.java similarity index 96% rename from java/src/main/java/org/rocksdb/BuiltinComparator.java rename to java/src/main/java/org/forstdb/BuiltinComparator.java index 2c89bf218..89faa1611 100644 --- a/java/src/main/java/org/rocksdb/BuiltinComparator.java +++ b/java/src/main/java/org/forstdb/BuiltinComparator.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Builtin RocksDB comparators diff --git a/java/src/main/java/org/rocksdb/ByteBufferGetStatus.java b/java/src/main/java/org/forstdb/ByteBufferGetStatus.java similarity index 96% rename from java/src/main/java/org/rocksdb/ByteBufferGetStatus.java rename to java/src/main/java/org/forstdb/ByteBufferGetStatus.java index f918a8d03..4ead43026 100644 --- a/java/src/main/java/org/rocksdb/ByteBufferGetStatus.java +++ b/java/src/main/java/org/forstdb/ByteBufferGetStatus.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; import java.util.List; @@ -42,6 +42,7 @@ public class ByteBufferGetStatus { * * @param status the status of the request to fetch into the buffer */ + @SuppressWarnings("PMD.NullAssignment") ByteBufferGetStatus(final Status status) { this.status = status; this.requiredSize = 0; diff --git a/java/src/main/java/org/rocksdb/Cache.java b/java/src/main/java/org/forstdb/Cache.java similarity index 97% rename from java/src/main/java/org/rocksdb/Cache.java rename to java/src/main/java/org/forstdb/Cache.java index 04bd3fcaa..3db3ef10f 100644 --- a/java/src/main/java/org/rocksdb/Cache.java +++ b/java/src/main/java/org/forstdb/Cache.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public abstract class Cache extends RocksObject { diff --git a/java/src/main/java/org/rocksdb/CassandraCompactionFilter.java b/java/src/main/java/org/forstdb/CassandraCompactionFilter.java similarity index 97% rename from java/src/main/java/org/rocksdb/CassandraCompactionFilter.java rename to java/src/main/java/org/forstdb/CassandraCompactionFilter.java index 12854c510..58fc0c7f2 100644 --- a/java/src/main/java/org/rocksdb/CassandraCompactionFilter.java +++ b/java/src/main/java/org/forstdb/CassandraCompactionFilter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Just a Java wrapper around CassandraCompactionFilter implemented in C++ diff --git a/java/src/main/java/org/rocksdb/CassandraValueMergeOperator.java b/java/src/main/java/org/forstdb/CassandraValueMergeOperator.java similarity index 97% rename from java/src/main/java/org/rocksdb/CassandraValueMergeOperator.java rename to java/src/main/java/org/forstdb/CassandraValueMergeOperator.java index 732faee20..202a59c55 100644 --- a/java/src/main/java/org/rocksdb/CassandraValueMergeOperator.java +++ b/java/src/main/java/org/forstdb/CassandraValueMergeOperator.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * CassandraValueMergeOperator is a merge operator that merges two cassandra wide column diff --git a/java/src/main/java/org/rocksdb/Checkpoint.java b/java/src/main/java/org/forstdb/Checkpoint.java similarity index 88% rename from java/src/main/java/org/rocksdb/Checkpoint.java rename to java/src/main/java/org/forstdb/Checkpoint.java index 318d11c64..47bc74294 100644 --- a/java/src/main/java/org/rocksdb/Checkpoint.java +++ b/java/src/main/java/org/forstdb/Checkpoint.java @@ -3,9 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; - -import java.util.*; +package org.forstdb; /** * Provides Checkpoint functionality. Checkpoints @@ -54,7 +52,8 @@ public void createCheckpoint(final String checkpointPath) public ExportImportFilesMetaData exportColumnFamily(final ColumnFamilyHandle columnFamilyHandle, final String exportPath) throws RocksDBException { - return exportColumnFamily(nativeHandle_, columnFamilyHandle.nativeHandle_, exportPath); + return new ExportImportFilesMetaData( + exportColumnFamily(nativeHandle_, columnFamilyHandle.nativeHandle_, exportPath)); } private Checkpoint(final RocksDB db) { @@ -67,6 +66,6 @@ private Checkpoint(final RocksDB db) { private native void createCheckpoint(long handle, String checkpointPath) throws RocksDBException; - private native ExportImportFilesMetaData exportColumnFamily( - long handle, long columnFamilyHandle, String exportPath) throws RocksDBException; + private native long exportColumnFamily(long handle, long columnFamilyHandle, String exportPath) + throws RocksDBException; } diff --git a/java/src/main/java/org/rocksdb/ChecksumType.java b/java/src/main/java/org/forstdb/ChecksumType.java similarity index 92% rename from java/src/main/java/org/rocksdb/ChecksumType.java rename to java/src/main/java/org/forstdb/ChecksumType.java index e03fa14ba..8623f2541 100644 --- a/java/src/main/java/org/rocksdb/ChecksumType.java +++ b/java/src/main/java/org/forstdb/ChecksumType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Checksum types used in conjunction with BlockBasedTable. @@ -37,7 +37,7 @@ public byte getValue() { return value_; } - private ChecksumType(final byte value) { + ChecksumType(final byte value) { value_ = value; } diff --git a/java/src/main/java/org/rocksdb/ClockCache.java b/java/src/main/java/org/forstdb/ClockCache.java similarity index 79% rename from java/src/main/java/org/rocksdb/ClockCache.java rename to java/src/main/java/org/forstdb/ClockCache.java index e4251db8e..4ed6f7077 100644 --- a/java/src/main/java/org/rocksdb/ClockCache.java +++ b/java/src/main/java/org/forstdb/ClockCache.java @@ -3,17 +3,23 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Similar to {@link LRUCache}, but based on the CLOCK algorithm with * better concurrent performance in some cases + * + * @deprecated The old Clock Cache implementation had an unresolved bug and + * has been removed. The new HyperClockCache requires an additional + * configuration parameter that is not provided by this API. This function + * simply returns a new LRUCache for functional compatibility. */ public class ClockCache extends Cache { - /** * Create a new cache with a fixed size capacity. * + * @deprecated The old Clock Cache implementation had an unresolved bug and has been removed. + * * @param capacity The fixed size capacity of the cache */ public ClockCache(final long capacity) { @@ -27,6 +33,8 @@ public ClockCache(final long capacity) { * numShardBits = -1 means it is automatically determined: every shard * will be at least 512KB and number of shard bits will not exceed 6. * + * @deprecated The old Clock Cache implementation had an unresolved bug and has been removed. + * * @param capacity The fixed size capacity of the cache * @param numShardBits The cache is sharded to 2^numShardBits shards, * by hash of the key @@ -43,6 +51,8 @@ public ClockCache(final long capacity, final int numShardBits) { * numShardBits = -1 means it is automatically determined: every shard * will be at least 512KB and number of shard bits will not exceed 6. * + * @deprecated The old Clock Cache implementation had an unresolved bug and has been removed. + * * @param capacity The fixed size capacity of the cache * @param numShardBits The cache is sharded to 2^numShardBits shards, * by hash of the key diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyDescriptor.java b/java/src/main/java/org/forstdb/ColumnFamilyDescriptor.java similarity index 89% rename from java/src/main/java/org/rocksdb/ColumnFamilyDescriptor.java rename to java/src/main/java/org/forstdb/ColumnFamilyDescriptor.java index 125a8dcf8..ec2c99fa0 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyDescriptor.java +++ b/java/src/main/java/org/forstdb/ColumnFamilyDescriptor.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Arrays; @@ -33,8 +33,9 @@ public ColumnFamilyDescriptor(final byte[] columnFamilyName) { * column family. * @since 3.10.0 */ - public ColumnFamilyDescriptor(final byte[] columnFamilyName, - final ColumnFamilyOptions columnFamilyOptions) { + @SuppressWarnings("PMD.ArrayIsStoredDirectly") + public ColumnFamilyDescriptor( + final byte[] columnFamilyName, final ColumnFamilyOptions columnFamilyOptions) { columnFamilyName_ = columnFamilyName; columnFamilyOptions_ = columnFamilyOptions; } @@ -45,6 +46,7 @@ public ColumnFamilyDescriptor(final byte[] columnFamilyName, * @return column family name. * @since 3.10.0 */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public byte[] getName() { return columnFamilyName_; } diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java b/java/src/main/java/org/forstdb/ColumnFamilyHandle.java similarity index 97% rename from java/src/main/java/org/rocksdb/ColumnFamilyHandle.java rename to java/src/main/java/org/forstdb/ColumnFamilyHandle.java index 32ea4b04d..4f07375bd 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java +++ b/java/src/main/java/org/forstdb/ColumnFamilyHandle.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Arrays; import java.util.Objects; @@ -102,7 +102,7 @@ public boolean equals(final Object o) { return false; } - final ColumnFamilyHandle that = (ColumnFamilyHandle) o; + @SuppressWarnings("PMD.CloseResource") final ColumnFamilyHandle that = (ColumnFamilyHandle) o; try { return rocksDB_.nativeHandle_ == that.rocksDB_.nativeHandle_ && getID() == that.getID() && diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyMetaData.java b/java/src/main/java/org/forstdb/ColumnFamilyMetaData.java similarity index 91% rename from java/src/main/java/org/rocksdb/ColumnFamilyMetaData.java rename to java/src/main/java/org/forstdb/ColumnFamilyMetaData.java index 191904017..7c2734ec4 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyMetaData.java +++ b/java/src/main/java/org/forstdb/ColumnFamilyMetaData.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Arrays; import java.util.List; @@ -11,6 +11,7 @@ /** * The metadata that describes a column family. */ +@SuppressWarnings("PMD.MissingStaticMethodInNonInstantiatableClass") public class ColumnFamilyMetaData { private final long size; private final long fileCount; @@ -55,6 +56,7 @@ public long fileCount() { * * @return the name */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public byte[] name() { return name; } diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/java/src/main/java/org/forstdb/ColumnFamilyOptions.java similarity index 97% rename from java/src/main/java/org/rocksdb/ColumnFamilyOptions.java rename to java/src/main/java/org/forstdb/ColumnFamilyOptions.java index d8d9658fc..a7105806b 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java +++ b/java/src/main/java/org/forstdb/ColumnFamilyOptions.java @@ -3,25 +3,21 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.file.Paths; import java.util.*; /** * ColumnFamilyOptions to control the behavior of a database. It will be used - * during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()). + * during the creation of a {@link org.forstdb.RocksDB} (i.e., RocksDB.open()). *

* As a descendant of {@link AbstractNativeReference}, this class is {@link AutoCloseable} * and will be automatically released if opened in the preamble of a try with resources block. */ -public class ColumnFamilyOptions extends RocksObject - implements ColumnFamilyOptionsInterface, - MutableColumnFamilyOptionsInterface { - static { - RocksDB.loadLibrary(); - } - +public class ColumnFamilyOptions + extends RocksObject implements ColumnFamilyOptionsInterface, + MutableColumnFamilyOptionsInterface { /** * Construct ColumnFamilyOptions. *

@@ -29,7 +25,7 @@ public class ColumnFamilyOptions extends RocksObject * an {@code rocksdb::ColumnFamilyOptions} in the c++ side. */ public ColumnFamilyOptions() { - super(newColumnFamilyOptions()); + super(newColumnFamilyOptionsInstance()); } /** @@ -89,7 +85,7 @@ public ColumnFamilyOptions(final Options options) { * * @param properties {@link java.util.Properties} instance. * - * @return {@link org.rocksdb.ColumnFamilyOptions instance} + * @return {@link org.forstdb.ColumnFamilyOptions instance} * or null. * * @throws java.lang.IllegalArgumentException if null or empty @@ -120,7 +116,7 @@ public static ColumnFamilyOptions getColumnFamilyOptionsFromProps( * @param cfgOpts ConfigOptions controlling how the properties are parsed. * @param properties {@link java.util.Properties} instance. * - * @return {@link org.rocksdb.ColumnFamilyOptions instance} + * @return {@link org.forstdb.ColumnFamilyOptions instance} * or null. * * @throws java.lang.IllegalArgumentException if null or empty @@ -602,6 +598,10 @@ public ColumnFamilyOptions setTableFormatConfig( return this; } + void setFetchedTableFormatConfig(final TableFormatConfig tableFormatConfig) { + this.tableFormatConfig_ = tableFormatConfig; + } + @Override public String tableFactoryName() { assert(isOwningHandle()); @@ -959,6 +959,17 @@ public SstPartitionerFactory sstPartitionerFactory() { return sstPartitionerFactory_; } + @Override + public ColumnFamilyOptions setMemtableMaxRangeDeletions(final int count) { + setMemtableMaxRangeDeletions(nativeHandle_, count); + return this; + } + + @Override + public int memtableMaxRangeDeletions() { + return memtableMaxRangeDeletions(nativeHandle_); + } + // // BEGIN options for blobs (integrated BlobDB) // @@ -1001,6 +1012,7 @@ public ColumnFamilyOptions setEnableBlobFiles(final boolean enableBlobFiles) { * * @return true iff blob files are currently enabled */ + @Override public boolean enableBlobFiles() { return enableBlobFiles(nativeHandle_); } @@ -1322,6 +1334,10 @@ private static native long getColumnFamilyOptionsFromProps( final long cfgHandle, String optString); private static native long getColumnFamilyOptionsFromProps(final String optString); + private static long newColumnFamilyOptionsInstance() { + RocksDB.loadLibrary(); + return newColumnFamilyOptions(); + } private static native long newColumnFamilyOptions(); private static native long copyColumnFamilyOptions(final long handle); private static native long newColumnFamilyOptionsFromOptions( @@ -1498,7 +1514,8 @@ private native void setForceConsistencyChecks(final long handle, private native void setSstPartitionerFactory(long nativeHandle_, long newFactoryHandle); private static native void setCompactionThreadLimiter( final long nativeHandle_, final long compactionThreadLimiterHandle); - + private native void setMemtableMaxRangeDeletions(final long handle, final int count); + private native int memtableMaxRangeDeletions(final long handle); private native void setEnableBlobFiles(final long nativeHandle_, final boolean enableBlobFiles); private native boolean enableBlobFiles(final long nativeHandle_); private native void setMinBlobSize(final long nativeHandle_, final long minBlobSize); diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java b/java/src/main/java/org/forstdb/ColumnFamilyOptionsInterface.java similarity index 96% rename from java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java rename to java/src/main/java/org/forstdb/ColumnFamilyOptionsInterface.java index 776fc7038..06db0ffeb 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/forstdb/ColumnFamilyOptionsInterface.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Collection; import java.util.List; @@ -370,7 +370,7 @@ T setMaxTableFilesSizeFIFO( * Memtable format can be set using setTableFormatConfig. * * @return the name of the currently-used memtable factory. - * @see #setTableFormatConfig(org.rocksdb.TableFormatConfig) + * @see #setTableFormatConfig(org.forstdb.TableFormatConfig) */ String memTableFactoryName(); @@ -506,6 +506,23 @@ T setCompressionOptions( @Experimental("Caution: this option is experimental") SstPartitionerFactory sstPartitionerFactory(); + /** + * Sets the maximum range delete calls, after which memtable is flushed. + * This applies to the mutable memtable. + * + * @param count a positive integer, 0 (default) to disable the feature. + * @return the reference of the current options. + */ + T setMemtableMaxRangeDeletions(final int count); + + /** + * Gets the current setting of maximum range deletes allowed + * 0(default) indicates that feature is disabled. + * + * @return current value of memtable_max_range_deletions + */ + int memtableMaxRangeDeletions(); + /** * Compaction concurrent thread limiter for the column family. * If non-nullptr, use given concurrent thread limiter to control diff --git a/java/src/main/java/org/rocksdb/CompactRangeOptions.java b/java/src/main/java/org/forstdb/CompactRangeOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompactRangeOptions.java rename to java/src/main/java/org/forstdb/CompactRangeOptions.java index 616a77572..823be4e1f 100644 --- a/java/src/main/java/org/rocksdb/CompactRangeOptions.java +++ b/java/src/main/java/org/forstdb/CompactRangeOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/CompactionJobInfo.java b/java/src/main/java/org/forstdb/CompactionJobInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompactionJobInfo.java rename to java/src/main/java/org/forstdb/CompactionJobInfo.java index cf04bde24..2a5c8c4f5 100644 --- a/java/src/main/java/org/rocksdb/CompactionJobInfo.java +++ b/java/src/main/java/org/forstdb/CompactionJobInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Arrays; import java.util.List; diff --git a/java/src/main/java/org/rocksdb/CompactionJobStats.java b/java/src/main/java/org/forstdb/CompactionJobStats.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompactionJobStats.java rename to java/src/main/java/org/forstdb/CompactionJobStats.java index 3d53b5565..a10998a3a 100644 --- a/java/src/main/java/org/rocksdb/CompactionJobStats.java +++ b/java/src/main/java/org/forstdb/CompactionJobStats.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class CompactionJobStats extends RocksObject { diff --git a/java/src/main/java/org/rocksdb/CompactionOptions.java b/java/src/main/java/org/forstdb/CompactionOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompactionOptions.java rename to java/src/main/java/org/forstdb/CompactionOptions.java index 2c7e391fb..69b11eb87 100644 --- a/java/src/main/java/org/rocksdb/CompactionOptions.java +++ b/java/src/main/java/org/forstdb/CompactionOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.List; diff --git a/java/src/main/java/org/rocksdb/CompactionOptionsFIFO.java b/java/src/main/java/org/forstdb/CompactionOptionsFIFO.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompactionOptionsFIFO.java rename to java/src/main/java/org/forstdb/CompactionOptionsFIFO.java index 92b21fc50..0b321a32e 100644 --- a/java/src/main/java/org/rocksdb/CompactionOptionsFIFO.java +++ b/java/src/main/java/org/forstdb/CompactionOptionsFIFO.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Options for FIFO Compaction diff --git a/java/src/main/java/org/rocksdb/CompactionOptionsUniversal.java b/java/src/main/java/org/forstdb/CompactionOptionsUniversal.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompactionOptionsUniversal.java rename to java/src/main/java/org/forstdb/CompactionOptionsUniversal.java index 4d2ebdb1f..10e974e43 100644 --- a/java/src/main/java/org/rocksdb/CompactionOptionsUniversal.java +++ b/java/src/main/java/org/forstdb/CompactionOptionsUniversal.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Options for Universal Compaction diff --git a/java/src/main/java/org/rocksdb/CompactionPriority.java b/java/src/main/java/org/forstdb/CompactionPriority.java similarity index 96% rename from java/src/main/java/org/rocksdb/CompactionPriority.java rename to java/src/main/java/org/forstdb/CompactionPriority.java index eda05942e..1ba172dfc 100644 --- a/java/src/main/java/org/rocksdb/CompactionPriority.java +++ b/java/src/main/java/org/forstdb/CompactionPriority.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Compaction Priorities @@ -64,7 +64,7 @@ public byte getValue() { * * @param value byte representation of CompactionPriority. * - * @return {@link org.rocksdb.CompactionPriority} instance or null. + * @return {@link org.forstdb.CompactionPriority} instance or null. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/CompactionReason.java b/java/src/main/java/org/forstdb/CompactionReason.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompactionReason.java rename to java/src/main/java/org/forstdb/CompactionReason.java index 46ec33f3f..4e6b19860 100644 --- a/java/src/main/java/org/rocksdb/CompactionReason.java +++ b/java/src/main/java/org/forstdb/CompactionReason.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum CompactionReason { kUnknown((byte)0x0), diff --git a/java/src/main/java/org/rocksdb/CompactionStopStyle.java b/java/src/main/java/org/forstdb/CompactionStopStyle.java similarity index 93% rename from java/src/main/java/org/rocksdb/CompactionStopStyle.java rename to java/src/main/java/org/forstdb/CompactionStopStyle.java index f6e63209c..fe1abf2a0 100644 --- a/java/src/main/java/org/rocksdb/CompactionStopStyle.java +++ b/java/src/main/java/org/forstdb/CompactionStopStyle.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** * Algorithm used to make a compaction request stop picking new files @@ -38,7 +38,7 @@ public byte getValue() { * * @param value byte representation of CompactionStopStyle. * - * @return {@link org.rocksdb.CompactionStopStyle} instance or null. + * @return {@link org.forstdb.CompactionStopStyle} instance or null. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/CompactionStyle.java b/java/src/main/java/org/forstdb/CompactionStyle.java similarity index 97% rename from java/src/main/java/org/rocksdb/CompactionStyle.java rename to java/src/main/java/org/forstdb/CompactionStyle.java index 794074df6..cf6047e26 100644 --- a/java/src/main/java/org/rocksdb/CompactionStyle.java +++ b/java/src/main/java/org/forstdb/CompactionStyle.java @@ -3,9 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; - -import java.util.List; +package org.forstdb; /** * Enum CompactionStyle @@ -25,7 +23,7 @@ * the old data, so it's basically a TTL compaction style. *

  • NONE - Disable background compaction. * Compaction jobs are submitted - * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, List, int, int, + * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, java.util.List, int, int, * CompactionJobInfo)} ()}.
    • * * diff --git a/java/src/main/java/org/rocksdb/ComparatorOptions.java b/java/src/main/java/org/forstdb/ComparatorOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/ComparatorOptions.java rename to java/src/main/java/org/forstdb/ComparatorOptions.java index ee5beb8f6..d14ffc095 100644 --- a/java/src/main/java/org/rocksdb/ComparatorOptions.java +++ b/java/src/main/java/org/forstdb/ComparatorOptions.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * This class controls the behaviour diff --git a/java/src/main/java/org/rocksdb/ComparatorType.java b/java/src/main/java/org/forstdb/ComparatorType.java similarity index 98% rename from java/src/main/java/org/rocksdb/ComparatorType.java rename to java/src/main/java/org/forstdb/ComparatorType.java index 199980b6e..a2585ecfc 100644 --- a/java/src/main/java/org/rocksdb/ComparatorType.java +++ b/java/src/main/java/org/forstdb/ComparatorType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; enum ComparatorType { JAVA_COMPARATOR((byte)0x0), diff --git a/java/src/main/java/org/rocksdb/CompressionOptions.java b/java/src/main/java/org/forstdb/CompressionOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompressionOptions.java rename to java/src/main/java/org/forstdb/CompressionOptions.java index 2e1ee5731..2cce1622a 100644 --- a/java/src/main/java/org/rocksdb/CompressionOptions.java +++ b/java/src/main/java/org/forstdb/CompressionOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Options for Compression diff --git a/java/src/main/java/org/rocksdb/CompressionType.java b/java/src/main/java/org/forstdb/CompressionType.java similarity index 99% rename from java/src/main/java/org/rocksdb/CompressionType.java rename to java/src/main/java/org/forstdb/CompressionType.java index d1ecf0ac8..52003386d 100644 --- a/java/src/main/java/org/rocksdb/CompressionType.java +++ b/java/src/main/java/org/forstdb/CompressionType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Enum CompressionType diff --git a/java/src/main/java/org/rocksdb/ConcurrentTaskLimiter.java b/java/src/main/java/org/forstdb/ConcurrentTaskLimiter.java similarity index 98% rename from java/src/main/java/org/rocksdb/ConcurrentTaskLimiter.java rename to java/src/main/java/org/forstdb/ConcurrentTaskLimiter.java index b4e34303b..ee3d854c5 100644 --- a/java/src/main/java/org/rocksdb/ConcurrentTaskLimiter.java +++ b/java/src/main/java/org/forstdb/ConcurrentTaskLimiter.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public abstract class ConcurrentTaskLimiter extends RocksObject { protected ConcurrentTaskLimiter(final long nativeHandle) { diff --git a/java/src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java b/java/src/main/java/org/forstdb/ConcurrentTaskLimiterImpl.java similarity index 98% rename from java/src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java rename to java/src/main/java/org/forstdb/ConcurrentTaskLimiterImpl.java index d28b9060a..b41f5e1d1 100644 --- a/java/src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java +++ b/java/src/main/java/org/forstdb/ConcurrentTaskLimiterImpl.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class ConcurrentTaskLimiterImpl extends ConcurrentTaskLimiter { public ConcurrentTaskLimiterImpl(final String name, final int maxOutstandingTask) { diff --git a/java/src/main/java/org/rocksdb/ConfigOptions.java b/java/src/main/java/org/forstdb/ConfigOptions.java similarity index 92% rename from java/src/main/java/org/rocksdb/ConfigOptions.java rename to java/src/main/java/org/forstdb/ConfigOptions.java index 026f8b01d..0ea711e3d 100644 --- a/java/src/main/java/org/rocksdb/ConfigOptions.java +++ b/java/src/main/java/org/forstdb/ConfigOptions.java @@ -4,18 +4,14 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class ConfigOptions extends RocksObject { - static { - RocksDB.loadLibrary(); - } - /** * Construct with default Options */ public ConfigOptions() { - super(newConfigOptions()); + super(newConfigOptionsInstance()); } public ConfigOptions setDelimiter(final String delimiter) { @@ -44,6 +40,10 @@ public ConfigOptions setSanityLevel(final SanityLevel level) { @Override protected final native void disposeInternal(final long handle); + private static long newConfigOptionsInstance() { + RocksDB.loadLibrary(); + return newConfigOptions(); + } private static native long newConfigOptions(); private static native void setEnv(final long handle, final long envHandle); private static native void setDelimiter(final long handle, final String delimiter); diff --git a/java/src/main/java/org/rocksdb/DBOptions.java b/java/src/main/java/org/forstdb/DBOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/DBOptions.java rename to java/src/main/java/org/forstdb/DBOptions.java index 655d900c3..dd2722cdc 100644 --- a/java/src/main/java/org/rocksdb/DBOptions.java +++ b/java/src/main/java/org/forstdb/DBOptions.java @@ -3,25 +3,20 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.file.Paths; import java.util.*; /** * DBOptions to control the behavior of a database. It will be used - * during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()). + * during the creation of a {@link org.forstdb.RocksDB} (i.e., RocksDB.open()). *

    * As a descendent of {@link AbstractNativeReference}, this class is {@link AutoCloseable} * and will be automatically released if opened in the preamble of a try with resources block. */ public class DBOptions extends RocksObject - implements DBOptionsInterface, - MutableDBOptionsInterface { - static { - RocksDB.loadLibrary(); - } - + implements DBOptionsInterface, MutableDBOptionsInterface { /** * Construct DBOptions. *

    @@ -29,7 +24,7 @@ public class DBOptions extends RocksObject * an {@code rocksdb::DBOptions} in the c++ side. */ public DBOptions() { - super(newDBOptions()); + super(newDBOptionsInstance()); numShardBits_ = DEFAULT_NUM_SHARD_BITS; env_ = Env.getDefault(); } @@ -75,7 +70,7 @@ public DBOptions(final Options options) { * @param cfgOpts The ConfigOptions to control how the string is processed. * @param properties {@link java.util.Properties} instance. * - * @return {@link org.rocksdb.DBOptions instance} + * @return {@link org.forstdb.DBOptions instance} * or null. * * @throws java.lang.IllegalArgumentException if null or empty @@ -105,7 +100,7 @@ public static DBOptions getDBOptionsFromProps( * * @param properties {@link java.util.Properties} instance. * - * @return {@link org.rocksdb.DBOptions instance} + * @return {@link org.forstdb.DBOptions instance} * or null. * * @throws java.lang.IllegalArgumentException if null or empty @@ -752,6 +747,7 @@ public long dbWriteBufferSize() { } @Override + @Deprecated public DBOptions setAccessHintOnCompactionStart(final AccessHint accessHint) { assert(isOwningHandle()); setAccessHintOnCompactionStart(nativeHandle_, accessHint.getValue()); @@ -759,6 +755,7 @@ public DBOptions setAccessHintOnCompactionStart(final AccessHint accessHint) { } @Override + @Deprecated public AccessHint accessHintOnCompactionStart() { assert(isOwningHandle()); return AccessHint.getAccessHint(accessHintOnCompactionStart(nativeHandle_)); @@ -1251,7 +1248,12 @@ private DBOptions(final long nativeHandle) { private static native long getDBOptionsFromProps(long cfgHandle, String optString); private static native long getDBOptionsFromProps(String optString); + private static long newDBOptionsInstance() { + RocksDB.loadLibrary(); + return newDBOptions(); + } private static native long newDBOptions(); + private static native long copyDBOptions(final long handle); private static native long newDBOptionsFromOptions(final long optionsHandle); @Override protected final native void disposeInternal(final long handle); diff --git a/java/src/main/java/org/rocksdb/DBOptionsInterface.java b/java/src/main/java/org/forstdb/DBOptionsInterface.java similarity index 93% rename from java/src/main/java/org/rocksdb/DBOptionsInterface.java rename to java/src/main/java/org/forstdb/DBOptionsInterface.java index ef1b86bff..648502cc7 100644 --- a/java/src/main/java/org/rocksdb/DBOptionsInterface.java +++ b/java/src/main/java/org/forstdb/DBOptionsInterface.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Collection; import java.util.List; @@ -55,10 +55,10 @@ public interface DBOptionsInterface> { * Default: false * * @param flag a flag indicating whether to create a database the - * specified database in {@link RocksDB#open(org.rocksdb.Options, String)} operation + * specified database in {@link RocksDB#open(org.forstdb.Options, String)} operation * is missing. * @return the instance of the current Options - * @see RocksDB#open(org.rocksdb.Options, String) + * @see RocksDB#open(org.forstdb.Options, String) */ T setCreateIfMissing(boolean flag); @@ -101,7 +101,7 @@ public interface DBOptionsInterface> { * @param errorIfExists if true, an exception will be thrown * during {@code RocksDB.open()} if the database already exists. * @return the reference to the current option. - * @see RocksDB#open(org.rocksdb.Options, String) + * @see RocksDB#open(org.forstdb.Options, String) */ T setErrorIfExists(boolean errorIfExists); @@ -150,7 +150,7 @@ public interface DBOptionsInterface> { * priority than compaction. Rate limiting is disabled if nullptr. * Default: nullptr * - * @param rateLimiter {@link org.rocksdb.RateLimiter} instance. + * @param rateLimiter {@link org.forstdb.RateLimiter} instance. * @return the instance of the current object. * * @since 3.10.0 @@ -200,7 +200,7 @@ public interface DBOptionsInterface> { /** *

    Returns currently set log level.

    - * @return {@link org.rocksdb.InfoLogLevel} instance. + * @return {@link org.forstdb.InfoLogLevel} instance. */ InfoLogLevel infoLogLevel(); @@ -238,7 +238,7 @@ public interface DBOptionsInterface> { * * @return the instance of the current object. * - * @see RocksDB#open(org.rocksdb.Options, String) + * @see RocksDB#open(org.forstdb.Options, String) */ T setStatistics(final Statistics statistics); @@ -615,21 +615,24 @@ public interface DBOptionsInterface> { int tableCacheNumshardbits(); /** - * {@link #walTtlSeconds()} and {@link #walSizeLimitMB()} affect how archived logs - * will be deleted. - *
      - *
    1. If both set to 0, logs will be deleted asap and will not get into - * the archive.
      2. - *
    2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, - * WAL files will be checked every 10 min and if total size is greater - * then WAL_size_limit_MB, they will be deleted starting with the - * earliest until size_limit is met. All empty files will be deleted.
      3. - *
    3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then - * WAL files will be checked every WAL_ttl_seconds / 2 and those that - * are older than WAL_ttl_seconds will be deleted.
      4. - *
    4. If both are not 0, WAL files will be checked every 10 min and both - * checks will be performed with ttl being first.
      5. - *
    + * {@link #walTtlSeconds()} and {@link #walSizeLimitMB()} affect when WALs + * will be archived and deleted. + * + * When both are zero, obsolete WALs will not be archived and will be deleted + * immediately. Otherwise, obsolete WALs will be archived prior to deletion. + * + * When `WAL_size_limit_MB` is nonzero, archived WALs starting with the + * earliest will be deleted until the total size of the archive falls below + * this limit. All empty WALs will be deleted. + * + * When `WAL_ttl_seconds` is nonzero, archived WALs older than + * `WAL_ttl_seconds` will be deleted. + * + * When only `WAL_ttl_seconds` is nonzero, the frequency at which archived + * WALs are deleted is every `WAL_ttl_seconds / 2` seconds. When only + * `WAL_size_limit_MB` is nonzero, the deletion frequency is every ten + * minutes. When both are nonzero, the deletion frequency is the minimum of + * those two values. * * @param walTtlSeconds the ttl seconds * @return the instance of the current object. @@ -638,21 +641,24 @@ public interface DBOptionsInterface> { T setWalTtlSeconds(long walTtlSeconds); /** - * WalTtlSeconds() and walSizeLimitMB() affect how archived logs - * will be deleted. - *
      - *
    1. If both set to 0, logs will be deleted asap and will not get into - * the archive.
      2. - *
    2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, - * WAL files will be checked every 10 min and if total size is greater - * then WAL_size_limit_MB, they will be deleted starting with the - * earliest until size_limit is met. All empty files will be deleted.
      3. - *
    3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then - * WAL files will be checked every WAL_ttl_seconds / 2 and those that - * are older than WAL_ttl_seconds will be deleted.
      4. - *
    4. If both are not 0, WAL files will be checked every 10 min and both - * checks will be performed with ttl being first.
      5. - *
    + * WalTtlSeconds() and walSizeLimitMB() affect when WALs will be archived and + * deleted. + * + * When both are zero, obsolete WALs will not be archived and will be deleted + * immediately. Otherwise, obsolete WALs will be archived prior to deletion. + * + * When `WAL_size_limit_MB` is nonzero, archived WALs starting with the + * earliest will be deleted until the total size of the archive falls below + * this limit. All empty WALs will be deleted. + * + * When `WAL_ttl_seconds` is nonzero, archived WALs older than + * `WAL_ttl_seconds` will be deleted. + * + * When only `WAL_ttl_seconds` is nonzero, the frequency at which archived + * WALs are deleted is every `WAL_ttl_seconds / 2` seconds. When only + * `WAL_size_limit_MB` is nonzero, the deletion frequency is every ten + * minutes. When both are nonzero, the deletion frequency is the minimum of + * those two values. * * @return the wal-ttl seconds * @see #walSizeLimitMB() @@ -662,19 +668,22 @@ public interface DBOptionsInterface> { /** * WalTtlSeconds() and walSizeLimitMB() affect how archived logs * will be deleted. - *
      - *
    1. If both set to 0, logs will be deleted asap and will not get into - * the archive.
      2. - *
    2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, - * WAL files will be checked every 10 min and if total size is greater - * then WAL_size_limit_MB, they will be deleted starting with the - * earliest until size_limit is met. All empty files will be deleted.
      3. - *
    3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then - * WAL files will be checked every WAL_ttl_secondsi / 2 and those that - * are older than WAL_ttl_seconds will be deleted.
      4. - *
    4. If both are not 0, WAL files will be checked every 10 min and both - * checks will be performed with ttl being first.
      5. - *
    + * + * When both are zero, obsolete WALs will not be archived and will be deleted + * immediately. Otherwise, obsolete WALs will be archived prior to deletion. + * + * When `WAL_size_limit_MB` is nonzero, archived WALs starting with the + * earliest will be deleted until the total size of the archive falls below + * this limit. All empty WALs will be deleted. + * + * When `WAL_ttl_seconds` is nonzero, archived WALs older than + * `WAL_ttl_seconds` will be deleted. + * + * When only `WAL_ttl_seconds` is nonzero, the frequency at which archived + * WALs are deleted is every `WAL_ttl_seconds / 2` seconds. When only + * `WAL_size_limit_MB` is nonzero, the deletion frequency is every ten + * minutes. When both are nonzero, the deletion frequency is the minimum of + * those two values. * * @param sizeLimitMB size limit in mega-bytes. * @return the instance of the current object. @@ -683,21 +692,25 @@ public interface DBOptionsInterface> { T setWalSizeLimitMB(long sizeLimitMB); /** - * {@link #walTtlSeconds()} and {@code #walSizeLimitMB()} affect how archived logs - * will be deleted. - *
      - *
    1. If both set to 0, logs will be deleted asap and will not get into - * the archive.
      2. - *
    2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, - * WAL files will be checked every 10 min and if total size is greater - * then WAL_size_limit_MB, they will be deleted starting with the - * earliest until size_limit is met. All empty files will be deleted.
      3. - *
    3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then - * WAL files will be checked every WAL_ttl_seconds i / 2 and those that - * are older than WAL_ttl_seconds will be deleted.
      4. - *
    4. If both are not 0, WAL files will be checked every 10 min and both - * checks will be performed with ttl being first.
      5. - *
    + * WalTtlSeconds() and walSizeLimitMB() affect when WALs will be archived and + * deleted. + * + * When both are zero, obsolete WALs will not be archived and will be deleted + * immediately. Otherwise, obsolete WALs will be archived prior to deletion. + * + * When `WAL_size_limit_MB` is nonzero, archived WALs starting with the + * earliest will be deleted until the total size of the archive falls below + * this limit. All empty WALs will be deleted. + * + * When `WAL_ttl_seconds` is nonzero, archived WALs older than + * `WAL_ttl_seconds` will be deleted. + * + * When only `WAL_ttl_seconds` is nonzero, the frequency at which archived + * WALs are deleted is every `WAL_ttl_seconds / 2` seconds. When only + * `WAL_size_limit_MB` is nonzero, the deletion frequency is every ten + * minutes. When both are nonzero, the deletion frequency is the minimum of + * those two values. + * * @return size limit in mega-bytes. * @see #walSizeLimitMB() */ @@ -935,7 +948,7 @@ public interface DBOptionsInterface> { * * @return the reference to the current options. */ - T setAccessHintOnCompactionStart(final AccessHint accessHint); + @Deprecated T setAccessHintOnCompactionStart(final AccessHint accessHint); /** * Specify the file access pattern once a compaction is started. @@ -945,7 +958,7 @@ public interface DBOptionsInterface> { * * @return The access hint */ - AccessHint accessHintOnCompactionStart(); + @Deprecated AccessHint accessHintOnCompactionStart(); /** * This is a maximum buffer size that is used by WinMmapReadableFile in diff --git a/java/src/main/java/org/rocksdb/DataBlockIndexType.java b/java/src/main/java/org/forstdb/DataBlockIndexType.java similarity index 96% rename from java/src/main/java/org/rocksdb/DataBlockIndexType.java rename to java/src/main/java/org/forstdb/DataBlockIndexType.java index 513e5b429..446ee21e8 100644 --- a/java/src/main/java/org/rocksdb/DataBlockIndexType.java +++ b/java/src/main/java/org/forstdb/DataBlockIndexType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** diff --git a/java/src/main/java/org/rocksdb/DbPath.java b/java/src/main/java/org/forstdb/DbPath.java similarity index 98% rename from java/src/main/java/org/rocksdb/DbPath.java rename to java/src/main/java/org/forstdb/DbPath.java index 3f0b67557..b7d511206 100644 --- a/java/src/main/java/org/rocksdb/DbPath.java +++ b/java/src/main/java/org/forstdb/DbPath.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.file.Path; diff --git a/java/src/main/java/org/rocksdb/DirectSlice.java b/java/src/main/java/org/forstdb/DirectSlice.java similarity index 98% rename from java/src/main/java/org/rocksdb/DirectSlice.java rename to java/src/main/java/org/forstdb/DirectSlice.java index 5aa0866ff..af76395ef 100644 --- a/java/src/main/java/org/rocksdb/DirectSlice.java +++ b/java/src/main/java/org/forstdb/DirectSlice.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; @@ -13,7 +13,7 @@ *

    * ByteBuffer backed slices typically perform better with * larger keys and values. When using smaller keys and - * values consider using @see org.rocksdb.Slice + * values consider using @see org.forstdb.Slice */ public class DirectSlice extends AbstractSlice { public static final DirectSlice NONE = new DirectSlice(); diff --git a/java/src/main/java/org/rocksdb/EncodingType.java b/java/src/main/java/org/forstdb/EncodingType.java similarity index 96% rename from java/src/main/java/org/rocksdb/EncodingType.java rename to java/src/main/java/org/forstdb/EncodingType.java index c2790c195..77244d63d 100644 --- a/java/src/main/java/org/rocksdb/EncodingType.java +++ b/java/src/main/java/org/forstdb/EncodingType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * EncodingType @@ -47,7 +47,7 @@ public byte getValue() { return value_; } - private EncodingType(final byte value) { + EncodingType(final byte value) { value_ = value; } diff --git a/java/src/main/java/org/rocksdb/Env.java b/java/src/main/java/org/forstdb/Env.java similarity index 81% rename from java/src/main/java/org/rocksdb/Env.java rename to java/src/main/java/org/forstdb/Env.java index db4c6fd78..15b99fd7a 100644 --- a/java/src/main/java/org/rocksdb/Env.java +++ b/java/src/main/java/org/forstdb/Env.java @@ -3,29 +3,17 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; /** * Base class for all Env implementations in RocksDB. */ public abstract class Env extends RocksObject { - - static { - RocksDB.loadLibrary(); - } - - private static final Env DEFAULT_ENV = new RocksEnv(getDefaultEnvInternal()); - static { - /* - * The Ownership of the Default Env belongs to C++ - * and so we disown the native handle here so that - * we cannot accidentally free it from Java. - */ - DEFAULT_ENV.disOwnNativeHandle(); - } + private static final AtomicReference SINGULAR_DEFAULT_ENV = new AtomicReference<>(null); /** *

    Returns the default environment suitable for the current operating @@ -36,10 +24,34 @@ public abstract class Env extends RocksObject { * have the ownership of its c++ resource, and calling its dispose()/close() * will be no-op.

    * - * @return the default {@link org.rocksdb.RocksEnv} instance. + * @return the default {@link org.forstdb.RocksEnv} instance. */ + @SuppressWarnings({"PMD.CloseResource", "PMD.AssignmentInOperand"}) public static Env getDefault() { - return DEFAULT_ENV; + RocksEnv defaultEnv; + RocksEnv newDefaultEnv = null; + + while ((defaultEnv = SINGULAR_DEFAULT_ENV.get()) == null) { + // construct the RocksEnv only once in this thread + if (newDefaultEnv == null) { + // load the library just in-case it isn't already loaded! + RocksDB.loadLibrary(); + + newDefaultEnv = new RocksEnv(getDefaultEnvInternal()); + + /* + * The Ownership of the Default Env belongs to C++ + * and so we disown the native handle here so that + * we cannot accidentally free it from Java. + */ + newDefaultEnv.disOwnNativeHandle(); + } + + // use CAS to gracefully handle thread pre-emption + SINGULAR_DEFAULT_ENV.compareAndSet(null, newDefaultEnv); + } + + return defaultEnv; } /** diff --git a/java/src/main/java/org/rocksdb/EnvFlinkTestSuite.java b/java/src/main/java/org/forstdb/EnvFlinkTestSuite.java similarity index 98% rename from java/src/main/java/org/rocksdb/EnvFlinkTestSuite.java rename to java/src/main/java/org/forstdb/EnvFlinkTestSuite.java index 92e503509..469528059 100644 --- a/java/src/main/java/org/rocksdb/EnvFlinkTestSuite.java +++ b/java/src/main/java/org/forstdb/EnvFlinkTestSuite.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.rocksdb; +package org.forstdb; /** * The test suite used for flink-env interfaces testing. You could define and implement test diff --git a/java/src/main/java/org/rocksdb/EnvOptions.java b/java/src/main/java/org/forstdb/EnvOptions.java similarity index 98% rename from java/src/main/java/org/rocksdb/EnvOptions.java rename to java/src/main/java/org/forstdb/EnvOptions.java index 5cb193ac1..99c3005b8 100644 --- a/java/src/main/java/org/rocksdb/EnvOptions.java +++ b/java/src/main/java/org/forstdb/EnvOptions.java @@ -3,21 +3,17 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Options while opening a file to read/write */ public class EnvOptions extends RocksObject { - static { - RocksDB.loadLibrary(); - } - /** * Construct with default Options */ public EnvOptions() { - super(newEnvOptions()); + super(newEnvOptionsInstance()); } /** @@ -323,6 +319,10 @@ public RateLimiter rateLimiter() { return rateLimiter; } + private static long newEnvOptionsInstance() { + RocksDB.loadLibrary(); + return newEnvOptions(); + } private static native long newEnvOptions(); private static native long newEnvOptions(final long dboptions_handle); @Override protected final native void disposeInternal(final long handle); diff --git a/java/src/main/java/org/rocksdb/EventListener.java b/java/src/main/java/org/forstdb/EventListener.java similarity index 99% rename from java/src/main/java/org/rocksdb/EventListener.java rename to java/src/main/java/org/forstdb/EventListener.java index 27652eaf8..48244331b 100644 --- a/java/src/main/java/org/rocksdb/EventListener.java +++ b/java/src/main/java/org/forstdb/EventListener.java @@ -3,9 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; - -import java.util.List; +package org.forstdb; /** * EventListener class contains a set of callback functions that will @@ -16,14 +14,14 @@ * Note that callback functions should not run for an extended period of * time before the function returns, otherwise RocksDB may be blocked. * For example, it is not suggested to do - * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, List, int, int, + * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, java.util.List, int, int, * CompactionJobInfo)} (as it may run for a long while) or issue many of * {@link RocksDB#put(ColumnFamilyHandle, WriteOptions, byte[], byte[])} * (as Put may be blocked in certain cases) in the same thread in the * EventListener callback. *

    * However, doing - * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, List, int, int, + * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, java.util.List, int, int, * CompactionJobInfo)} and {@link RocksDB#put(ColumnFamilyHandle, WriteOptions, byte[], byte[])} in * another thread is considered safe. *

    diff --git a/java/src/main/java/org/rocksdb/Experimental.java b/java/src/main/java/org/forstdb/Experimental.java similarity index 97% rename from java/src/main/java/org/rocksdb/Experimental.java rename to java/src/main/java/org/forstdb/Experimental.java index 64b404d6f..b39272d3e 100644 --- a/java/src/main/java/org/rocksdb/Experimental.java +++ b/java/src/main/java/org/forstdb/Experimental.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.lang.annotation.ElementType; import java.lang.annotation.Documented; diff --git a/java/src/main/java/org/forstdb/ExportImportFilesMetaData.java b/java/src/main/java/org/forstdb/ExportImportFilesMetaData.java new file mode 100644 index 000000000..a2bac70c9 --- /dev/null +++ b/java/src/main/java/org/forstdb/ExportImportFilesMetaData.java @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +/** + * The metadata that describes a column family. + */ +public class ExportImportFilesMetaData extends RocksObject { + ExportImportFilesMetaData(final long nativeHandle) { + super(nativeHandle); + } + + @Override protected native void disposeInternal(final long handle); +} diff --git a/java/src/main/java/org/rocksdb/ExternalFileIngestionInfo.java b/java/src/main/java/org/forstdb/ExternalFileIngestionInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/ExternalFileIngestionInfo.java rename to java/src/main/java/org/forstdb/ExternalFileIngestionInfo.java index 7a99dd6bf..ab92c64d5 100644 --- a/java/src/main/java/org/rocksdb/ExternalFileIngestionInfo.java +++ b/java/src/main/java/org/forstdb/ExternalFileIngestionInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/FileOperationInfo.java b/java/src/main/java/org/forstdb/FileOperationInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/FileOperationInfo.java rename to java/src/main/java/org/forstdb/FileOperationInfo.java index fae9cd5de..9f3f8d50a 100644 --- a/java/src/main/java/org/rocksdb/FileOperationInfo.java +++ b/java/src/main/java/org/forstdb/FileOperationInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/Filter.java b/java/src/main/java/org/forstdb/Filter.java similarity index 98% rename from java/src/main/java/org/rocksdb/Filter.java rename to java/src/main/java/org/forstdb/Filter.java index 7f490cf59..94cd7e527 100644 --- a/java/src/main/java/org/rocksdb/Filter.java +++ b/java/src/main/java/org/forstdb/Filter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Filters are stored in rocksdb and are consulted automatically diff --git a/java/src/main/java/org/forstdb/FilterPolicyType.java b/java/src/main/java/org/forstdb/FilterPolicyType.java new file mode 100644 index 000000000..36621e152 --- /dev/null +++ b/java/src/main/java/org/forstdb/FilterPolicyType.java @@ -0,0 +1,49 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +/** + * IndexType used in conjunction with BlockBasedTable. + */ +public enum FilterPolicyType { + kUnknownFilterPolicy((byte) 0), + + /** + * This is a user-facing policy that automatically choose between + * LegacyBloom and FastLocalBloom based on context at build time, + * including compatibility with format_version. + */ + kBloomFilterPolicy((byte) 1), + + /** + * This is a user-facing policy that chooses between Standard128Ribbon + * and FastLocalBloom based on context at build time (LSM level and other + * factors in extreme cases). + */ + kRibbonFilterPolicy((byte) 2); + + public Filter createFilter(final long handle, final double param) { + if (this == kBloomFilterPolicy) { + return new BloomFilter(handle, param); + } + return null; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value_; + } + + FilterPolicyType(byte value) { + value_ = value; + } + + private final byte value_; +} diff --git a/java/src/main/java/org/rocksdb/FlinkCompactionFilter.java b/java/src/main/java/org/forstdb/FlinkCompactionFilter.java similarity index 99% rename from java/src/main/java/org/rocksdb/FlinkCompactionFilter.java rename to java/src/main/java/org/forstdb/FlinkCompactionFilter.java index ee575d5ba..40e867b23 100644 --- a/java/src/main/java/org/rocksdb/FlinkCompactionFilter.java +++ b/java/src/main/java/org/forstdb/FlinkCompactionFilter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Just a Java wrapper around FlinkCompactionFilter implemented in C++. diff --git a/java/src/main/java/org/rocksdb/FlinkEnv.java b/java/src/main/java/org/forstdb/FlinkEnv.java similarity index 98% rename from java/src/main/java/org/rocksdb/FlinkEnv.java rename to java/src/main/java/org/forstdb/FlinkEnv.java index 91e6d46b6..758e72952 100644 --- a/java/src/main/java/org/rocksdb/FlinkEnv.java +++ b/java/src/main/java/org/forstdb/FlinkEnv.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.rocksdb; +package org.forstdb; /** * Flink Env which proxy all filesystem access to Flink FileSystem. diff --git a/java/src/main/java/org/rocksdb/FlushJobInfo.java b/java/src/main/java/org/forstdb/FlushJobInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/FlushJobInfo.java rename to java/src/main/java/org/forstdb/FlushJobInfo.java index 414d3a2f3..c58f5d3f0 100644 --- a/java/src/main/java/org/rocksdb/FlushJobInfo.java +++ b/java/src/main/java/org/forstdb/FlushJobInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/FlushOptions.java b/java/src/main/java/org/forstdb/FlushOptions.java similarity index 94% rename from java/src/main/java/org/rocksdb/FlushOptions.java rename to java/src/main/java/org/forstdb/FlushOptions.java index 0ec835089..27de9dfef 100644 --- a/java/src/main/java/org/rocksdb/FlushOptions.java +++ b/java/src/main/java/org/forstdb/FlushOptions.java @@ -3,22 +3,18 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * FlushOptions to be passed to flush operations of - * {@link org.rocksdb.RocksDB}. + * {@link org.forstdb.RocksDB}. */ public class FlushOptions extends RocksObject { - static { - RocksDB.loadLibrary(); - } - /** * Construct a new instance of FlushOptions. */ public FlushOptions(){ - super(newFlushOptions()); + super(newFlushOptionsInance()); } /** @@ -77,7 +73,10 @@ public boolean allowWriteStall() { assert(isOwningHandle()); return allowWriteStall(nativeHandle_); } - + private static long newFlushOptionsInance() { + RocksDB.loadLibrary(); + return newFlushOptions(); + } private static native long newFlushOptions(); @Override protected final native void disposeInternal(final long handle); diff --git a/java/src/main/java/org/rocksdb/FlushReason.java b/java/src/main/java/org/forstdb/FlushReason.java similarity index 98% rename from java/src/main/java/org/rocksdb/FlushReason.java rename to java/src/main/java/org/forstdb/FlushReason.java index 9d486cda1..093b97b12 100644 --- a/java/src/main/java/org/rocksdb/FlushReason.java +++ b/java/src/main/java/org/forstdb/FlushReason.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum FlushReason { OTHERS((byte) 0x00), diff --git a/java/src/main/java/org/forstdb/GetStatus.java b/java/src/main/java/org/forstdb/GetStatus.java new file mode 100644 index 000000000..2e82c13e1 --- /dev/null +++ b/java/src/main/java/org/forstdb/GetStatus.java @@ -0,0 +1,32 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +/** + * The result for a fetch + * and the total size of the object fetched. + * If the target of the fetch is not big enough, this may be bigger than the contents of the target. + */ +public class GetStatus { + public final Status status; + public final int requiredSize; + + /** + * Constructor used for success status, when the value is contained in the buffer + * + * @param status the status of the request to fetch into the buffer + * @param requiredSize the size of the data, which may be bigger than the buffer + */ + GetStatus(final Status status, final int requiredSize) { + this.status = status; + this.requiredSize = requiredSize; + } + + static GetStatus fromStatusCode(final Status.Code code, final int requiredSize) { + return new GetStatus(new Status(code, Status.SubCode.getSubCode((byte) 0), null), requiredSize); + } +} diff --git a/java/src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java b/java/src/main/java/org/forstdb/HashLinkedListMemTableConfig.java similarity index 98% rename from java/src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java rename to java/src/main/java/org/forstdb/HashLinkedListMemTableConfig.java index 4bc860d1c..0acb02a89 100644 --- a/java/src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java +++ b/java/src/main/java/org/forstdb/HashLinkedListMemTableConfig.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** * The config for hash linked list memtable representation @@ -15,7 +15,7 @@ * and post a warning in the LOG. */ public class HashLinkedListMemTableConfig extends MemTableConfig { - public static final long DEFAULT_BUCKET_COUNT = 50000; + public static final long DEFAULT_BUCKET_COUNT = 50_000; public static final long DEFAULT_HUGE_PAGE_TLB_SIZE = 0; public static final int DEFAULT_BUCKET_ENTRIES_LOG_THRES = 4096; public static final boolean diff --git a/java/src/main/java/org/rocksdb/HashSkipListMemTableConfig.java b/java/src/main/java/org/forstdb/HashSkipListMemTableConfig.java similarity index 97% rename from java/src/main/java/org/rocksdb/HashSkipListMemTableConfig.java rename to java/src/main/java/org/forstdb/HashSkipListMemTableConfig.java index 7cfa1c0df..cc2680121 100644 --- a/java/src/main/java/org/rocksdb/HashSkipListMemTableConfig.java +++ b/java/src/main/java/org/forstdb/HashSkipListMemTableConfig.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** * The config for hash skip-list mem-table representation. @@ -15,7 +15,7 @@ * and post a warning in the LOG. */ public class HashSkipListMemTableConfig extends MemTableConfig { - public static final int DEFAULT_BUCKET_COUNT = 1000000; + public static final int DEFAULT_BUCKET_COUNT = 1_000_000; public static final int DEFAULT_BRANCHING_FACTOR = 4; public static final int DEFAULT_HEIGHT = 4; diff --git a/java/src/main/java/org/rocksdb/HistogramData.java b/java/src/main/java/org/forstdb/HistogramData.java similarity index 98% rename from java/src/main/java/org/rocksdb/HistogramData.java rename to java/src/main/java/org/forstdb/HistogramData.java index 81d890883..439f89a19 100644 --- a/java/src/main/java/org/rocksdb/HistogramData.java +++ b/java/src/main/java/org/forstdb/HistogramData.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class HistogramData { private final double median_; diff --git a/java/src/main/java/org/rocksdb/HistogramType.java b/java/src/main/java/org/forstdb/HistogramType.java similarity index 93% rename from java/src/main/java/org/rocksdb/HistogramType.java rename to java/src/main/java/org/forstdb/HistogramType.java index aad0d9550..342c44991 100644 --- a/java/src/main/java/org/rocksdb/HistogramType.java +++ b/java/src/main/java/org/forstdb/HistogramType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum HistogramType { @@ -175,6 +175,16 @@ public enum HistogramType { FILE_READ_DB_OPEN_MICROS((byte) 0x3C), + FILE_READ_GET_MICROS((byte) 0x3D), + + FILE_READ_MULTIGET_MICROS((byte) 0x3E), + + FILE_READ_DB_ITERATOR_MICROS((byte) 0x3F), + + FILE_READ_VERIFY_DB_CHECKSUM_MICROS((byte) 0x40), + + FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS((byte) 0x41), + // 0x1F for backwards compatibility on current minor version. HISTOGRAM_ENUM_MAX((byte) 0x1F); @@ -198,7 +208,7 @@ public byte getValue() { * * @param value byte representation of HistogramType. * - * @return {@link org.rocksdb.HistogramType} instance. + * @return {@link org.forstdb.HistogramType} instance. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/Holder.java b/java/src/main/java/org/forstdb/Holder.java similarity index 97% rename from java/src/main/java/org/rocksdb/Holder.java rename to java/src/main/java/org/forstdb/Holder.java index 716a0bda0..ffe1759a4 100644 --- a/java/src/main/java/org/rocksdb/Holder.java +++ b/java/src/main/java/org/forstdb/Holder.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Simple instance reference wrapper. diff --git a/java/src/main/java/org/forstdb/HyperClockCache.java b/java/src/main/java/org/forstdb/HyperClockCache.java new file mode 100644 index 000000000..2ad072e39 --- /dev/null +++ b/java/src/main/java/org/forstdb/HyperClockCache.java @@ -0,0 +1,60 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +/** + * HyperClockCache - A lock-free Cache alternative for RocksDB block cache + * that offers much improved CPU efficiency vs. LRUCache under high parallel + * load or high contention, with some caveats: + *

      + *
    • + * Not a general Cache implementation: can only be used for + * BlockBasedTableOptions::block_cache, which RocksDB uses in a way that is + * compatible with HyperClockCache. + *
      • + *
    • + * Requires an extra tuning parameter: see estimated_entry_charge below. + * Similarly, substantially changing the capacity with SetCapacity could + * harm efficiency. -> EXPERIMENTAL: the tuning parameter can be set to 0 + * to find the appropriate balance automatically. + *
      • + *
    • + * Cache priorities are less aggressively enforced, which could cause + * cache dilution from long range scans (unless they use fill_cache=false). + *
      • + *
    • + * Can be worse for small caches, because if almost all of a cache shard is + * pinned (more likely with non-partitioned filters), then CLOCK eviction + * becomes very CPU intensive. + *
      • + *
    + */ +@Experimental("HyperClockCache is still experimental and this API may change in future.") +public class HyperClockCache extends Cache { + /** + * + * @param capacity The fixed size capacity of the cache + * @param estimatedEntryCharge EXPERIMENTAL: the field can be set to 0 to size the table + * dynamically and automatically. See C++ Api for more info. + * @param numShardBits The cache is sharded to 2^numShardBits shards, by hash of the key + * @param strictCapacityLimit insert to the cache will fail when cache is full + */ + public HyperClockCache(final long capacity, final long estimatedEntryCharge, int numShardBits, + boolean strictCapacityLimit) { + super(newHyperClockCache(capacity, estimatedEntryCharge, numShardBits, strictCapacityLimit)); + } + + @Override + protected void disposeInternal(long handle) { + disposeInternalJni(handle); + } + + private static native void disposeInternalJni(long handle); + + private static native long newHyperClockCache(final long capacity, + final long estimatedEntryCharge, int numShardBits, boolean strictCapacityLimit); +} diff --git a/java/src/main/java/org/rocksdb/ImportColumnFamilyOptions.java b/java/src/main/java/org/forstdb/ImportColumnFamilyOptions.java similarity index 85% rename from java/src/main/java/org/rocksdb/ImportColumnFamilyOptions.java rename to java/src/main/java/org/forstdb/ImportColumnFamilyOptions.java index acd89e92b..26e1c8db3 100644 --- a/java/src/main/java/org/rocksdb/ImportColumnFamilyOptions.java +++ b/java/src/main/java/org/forstdb/ImportColumnFamilyOptions.java @@ -4,14 +4,12 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** - * ImportColumnFamilyOptions is used by ImportColumnFamily() - *

    - * Note that dispose() must be called before this instance become out-of-scope - * to release the allocated memory in c++. - * + * ImportColumnFamilyOptions is used by + * {@link RocksDB#createColumnFamilyWithImport(ColumnFamilyDescriptor, ImportColumnFamilyOptions, + * ExportImportFilesMetaData)}. */ public class ImportColumnFamilyOptions extends RocksObject { public ImportColumnFamilyOptions() { diff --git a/java/src/main/java/org/rocksdb/IndexShorteningMode.java b/java/src/main/java/org/forstdb/IndexShorteningMode.java similarity index 99% rename from java/src/main/java/org/rocksdb/IndexShorteningMode.java rename to java/src/main/java/org/forstdb/IndexShorteningMode.java index a68346c38..bc2c79b83 100644 --- a/java/src/main/java/org/rocksdb/IndexShorteningMode.java +++ b/java/src/main/java/org/forstdb/IndexShorteningMode.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * This enum allows trading off increased index size for improved iterator diff --git a/java/src/main/java/org/rocksdb/IndexType.java b/java/src/main/java/org/forstdb/IndexType.java similarity index 98% rename from java/src/main/java/org/rocksdb/IndexType.java rename to java/src/main/java/org/forstdb/IndexType.java index 5615e929b..0e838fe51 100644 --- a/java/src/main/java/org/rocksdb/IndexType.java +++ b/java/src/main/java/org/forstdb/IndexType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * IndexType used in conjunction with BlockBasedTable. diff --git a/java/src/main/java/org/rocksdb/InfoLogLevel.java b/java/src/main/java/org/forstdb/InfoLogLevel.java similarity index 93% rename from java/src/main/java/org/rocksdb/InfoLogLevel.java rename to java/src/main/java/org/forstdb/InfoLogLevel.java index 197bd89da..3edbc5602 100644 --- a/java/src/main/java/org/rocksdb/InfoLogLevel.java +++ b/java/src/main/java/org/forstdb/InfoLogLevel.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** * RocksDB log levels. @@ -33,7 +33,7 @@ public byte getValue() { * * @param value byte representation of InfoLogLevel. * - * @return {@link org.rocksdb.InfoLogLevel} instance. + * @return {@link org.forstdb.InfoLogLevel} instance. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/IngestExternalFileOptions.java b/java/src/main/java/org/forstdb/IngestExternalFileOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/IngestExternalFileOptions.java rename to java/src/main/java/org/forstdb/IngestExternalFileOptions.java index 1a6a5fccd..7718c2082 100644 --- a/java/src/main/java/org/rocksdb/IngestExternalFileOptions.java +++ b/java/src/main/java/org/forstdb/IngestExternalFileOptions.java @@ -1,4 +1,4 @@ -package org.rocksdb; +package org.forstdb; // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License diff --git a/java/src/main/java/org/rocksdb/KeyMayExist.java b/java/src/main/java/org/forstdb/KeyMayExist.java similarity index 97% rename from java/src/main/java/org/rocksdb/KeyMayExist.java rename to java/src/main/java/org/forstdb/KeyMayExist.java index 6149b8529..31edabb99 100644 --- a/java/src/main/java/org/rocksdb/KeyMayExist.java +++ b/java/src/main/java/org/forstdb/KeyMayExist.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/LRUCache.java b/java/src/main/java/org/forstdb/LRUCache.java similarity index 99% rename from java/src/main/java/org/rocksdb/LRUCache.java rename to java/src/main/java/org/forstdb/LRUCache.java index 0a9d02e87..1799c2bfc 100644 --- a/java/src/main/java/org/rocksdb/LRUCache.java +++ b/java/src/main/java/org/forstdb/LRUCache.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Least Recently Used Cache diff --git a/java/src/main/java/org/rocksdb/LevelMetaData.java b/java/src/main/java/org/forstdb/LevelMetaData.java similarity index 93% rename from java/src/main/java/org/rocksdb/LevelMetaData.java rename to java/src/main/java/org/forstdb/LevelMetaData.java index c5685098b..28a0d3a89 100644 --- a/java/src/main/java/org/rocksdb/LevelMetaData.java +++ b/java/src/main/java/org/forstdb/LevelMetaData.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Arrays; import java.util.List; @@ -11,6 +11,7 @@ /** * The metadata that describes a level. */ +@SuppressWarnings("PMD.MissingStaticMethodInNonInstantiatableClass") public class LevelMetaData { private final int level; private final long size; diff --git a/java/src/main/java/org/rocksdb/LiveFileMetaData.java b/java/src/main/java/org/forstdb/LiveFileMetaData.java similarity index 73% rename from java/src/main/java/org/rocksdb/LiveFileMetaData.java rename to java/src/main/java/org/forstdb/LiveFileMetaData.java index 239dbf662..0b2af8b12 100644 --- a/java/src/main/java/org/rocksdb/LiveFileMetaData.java +++ b/java/src/main/java/org/forstdb/LiveFileMetaData.java @@ -3,11 +3,12 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The full set of metadata associated with each SST file. */ +@SuppressWarnings("PMD.MissingStaticMethodInNonInstantiatableClass") public class LiveFileMetaData extends SstFileMetaData { private final byte[] columnFamilyName; private final int level; @@ -15,22 +16,13 @@ public class LiveFileMetaData extends SstFileMetaData { /** * Called from JNI C++ */ - private LiveFileMetaData( - final byte[] columnFamilyName, - final int level, - final String fileName, - final String path, - final long size, - final long smallestSeqno, - final long largestSeqno, - final byte[] smallestKey, - final byte[] largestKey, - final long numReadsSampled, - final boolean beingCompacted, - final long numEntries, - final long numDeletions) { - super(fileName, path, size, smallestSeqno, largestSeqno, smallestKey, - largestKey, numReadsSampled, beingCompacted, numEntries, numDeletions); + private LiveFileMetaData(final byte[] columnFamilyName, final int level, final String fileName, + final String path, final long size, final long smallestSeqno, final long largestSeqno, + final byte[] smallestKey, final byte[] largestKey, final long numReadsSampled, + final boolean beingCompacted, final long numEntries, final long numDeletions, + final byte[] fileChecksum) { + super(fileName, path, size, smallestSeqno, largestSeqno, smallestKey, largestKey, + numReadsSampled, beingCompacted, numEntries, numDeletions, fileChecksum); this.columnFamilyName = columnFamilyName; this.level = level; } @@ -40,6 +32,7 @@ private LiveFileMetaData( * * @return the name of the column family */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public byte[] columnFamilyName() { return columnFamilyName; } diff --git a/java/src/main/java/org/rocksdb/LogFile.java b/java/src/main/java/org/forstdb/LogFile.java similarity index 95% rename from java/src/main/java/org/rocksdb/LogFile.java rename to java/src/main/java/org/forstdb/LogFile.java index ef24a6427..7a1503b77 100644 --- a/java/src/main/java/org/rocksdb/LogFile.java +++ b/java/src/main/java/org/forstdb/LogFile.java @@ -3,8 +3,9 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; +@SuppressWarnings("PMD.MissingStaticMethodInNonInstantiatableClass") public class LogFile { private final String pathName; private final long logNumber; diff --git a/java/src/main/java/org/rocksdb/Logger.java b/java/src/main/java/org/forstdb/Logger.java similarity index 84% rename from java/src/main/java/org/rocksdb/Logger.java rename to java/src/main/java/org/forstdb/Logger.java index 614a7fa50..deea5d740 100644 --- a/java/src/main/java/org/rocksdb/Logger.java +++ b/java/src/main/java/org/forstdb/Logger.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** *

    This class provides a custom logger functionality @@ -20,7 +20,7 @@ *

    * *

    - * A log level can be set using {@link org.rocksdb.Options} or + * A log level can be set using {@link org.forstdb.Options} or * {@link Logger#setInfoLogLevel(InfoLogLevel)}. The set log level * influences the underlying native code. Each log message is * checked against the set log level and if the log level is more @@ -31,8 +31,8 @@ *

    Every log message which will be emitted by native code will * trigger expensive native to Java transitions. So the preferred * setting for production use is either - * {@link org.rocksdb.InfoLogLevel#ERROR_LEVEL} or - * {@link org.rocksdb.InfoLogLevel#FATAL_LEVEL}. + * {@link org.forstdb.InfoLogLevel#ERROR_LEVEL} or + * {@link org.forstdb.InfoLogLevel#FATAL_LEVEL}. *

    */ public abstract class Logger extends RocksCallbackObject { @@ -43,10 +43,10 @@ public abstract class Logger extends RocksCallbackObject { *

    AbstractLogger constructor.

    * *

    Important: the log level set within - * the {@link org.rocksdb.Options} instance will be used as + * the {@link org.forstdb.Options} instance will be used as * maximum log level of RocksDB.

    * - * @param options {@link org.rocksdb.Options} instance. + * @param options {@link org.forstdb.Options} instance. */ public Logger(final Options options) { super(options.nativeHandle_, WITH_OPTIONS); @@ -57,10 +57,10 @@ public Logger(final Options options) { *

    AbstractLogger constructor.

    * *

    Important: the log level set within - * the {@link org.rocksdb.DBOptions} instance will be used + * the {@link org.forstdb.DBOptions} instance will be used * as maximum log level of RocksDB.

    * - * @param dboptions {@link org.rocksdb.DBOptions} instance. + * @param dboptions {@link org.forstdb.DBOptions} instance. */ public Logger(final DBOptions dboptions) { super(dboptions.nativeHandle_, WITH_DBOPTIONS); @@ -78,9 +78,9 @@ protected long initializeNative(final long... nativeParameterHandles) { } /** - * Set {@link org.rocksdb.InfoLogLevel} to AbstractLogger. + * Set {@link org.forstdb.InfoLogLevel} to AbstractLogger. * - * @param infoLogLevel {@link org.rocksdb.InfoLogLevel} instance. + * @param infoLogLevel {@link org.forstdb.InfoLogLevel} instance. */ public void setInfoLogLevel(final InfoLogLevel infoLogLevel) { setInfoLogLevel(nativeHandle_, infoLogLevel.getValue()); @@ -89,7 +89,7 @@ public void setInfoLogLevel(final InfoLogLevel infoLogLevel) { /** * Return the loggers log level. * - * @return {@link org.rocksdb.InfoLogLevel} instance. + * @return {@link org.forstdb.InfoLogLevel} instance. */ public InfoLogLevel infoLogLevel() { return InfoLogLevel.getInfoLogLevel( diff --git a/java/src/main/java/org/rocksdb/MemTableConfig.java b/java/src/main/java/org/forstdb/MemTableConfig.java similarity index 98% rename from java/src/main/java/org/rocksdb/MemTableConfig.java rename to java/src/main/java/org/forstdb/MemTableConfig.java index 17033d251..a9076b5aa 100644 --- a/java/src/main/java/org/rocksdb/MemTableConfig.java +++ b/java/src/main/java/org/forstdb/MemTableConfig.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * MemTableConfig is used to config the internal mem-table of a RocksDB. diff --git a/java/src/main/java/org/rocksdb/MemTableInfo.java b/java/src/main/java/org/forstdb/MemTableInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/MemTableInfo.java rename to java/src/main/java/org/forstdb/MemTableInfo.java index 3d429035a..8c738afe3 100644 --- a/java/src/main/java/org/rocksdb/MemTableInfo.java +++ b/java/src/main/java/org/forstdb/MemTableInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/MemoryUsageType.java b/java/src/main/java/org/forstdb/MemoryUsageType.java similarity index 98% rename from java/src/main/java/org/rocksdb/MemoryUsageType.java rename to java/src/main/java/org/forstdb/MemoryUsageType.java index 40e6d1716..18c707ac3 100644 --- a/java/src/main/java/org/rocksdb/MemoryUsageType.java +++ b/java/src/main/java/org/forstdb/MemoryUsageType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * MemoryUsageType diff --git a/java/src/main/java/org/rocksdb/MemoryUtil.java b/java/src/main/java/org/forstdb/MemoryUtil.java similarity index 94% rename from java/src/main/java/org/rocksdb/MemoryUtil.java rename to java/src/main/java/org/forstdb/MemoryUtil.java index 15b9f001a..01a87dada 100644 --- a/java/src/main/java/org/rocksdb/MemoryUtil.java +++ b/java/src/main/java/org/forstdb/MemoryUtil.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.*; @@ -27,7 +27,9 @@ public class MemoryUtil { * @param caches Set of caches to collect memory usage for. * @return Map from {@link MemoryUsageType} to memory usage as a {@link Long}. */ - public static Map getApproximateMemoryUsageByType(final List dbs, final Set caches) { + @SuppressWarnings("PMD.CloseResource") + public static Map getApproximateMemoryUsageByType( + final List dbs, final Set caches) { final int dbCount = (dbs == null) ? 0 : dbs.size(); final int cacheCount = (caches == null) ? 0 : caches.size(); final long[] dbHandles = new long[dbCount]; diff --git a/java/src/main/java/org/rocksdb/MergeOperator.java b/java/src/main/java/org/forstdb/MergeOperator.java similarity index 96% rename from java/src/main/java/org/rocksdb/MergeOperator.java rename to java/src/main/java/org/forstdb/MergeOperator.java index c299f6221..ea0430594 100644 --- a/java/src/main/java/org/rocksdb/MergeOperator.java +++ b/java/src/main/java/org/forstdb/MergeOperator.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * MergeOperator holds an operator to be applied when compacting diff --git a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java b/java/src/main/java/org/forstdb/MutableColumnFamilyOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java rename to java/src/main/java/org/forstdb/MutableColumnFamilyOptions.java index e54db7171..50b0fe8b1 100644 --- a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java +++ b/java/src/main/java/org/forstdb/MutableColumnFamilyOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.*; diff --git a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java b/java/src/main/java/org/forstdb/MutableColumnFamilyOptionsInterface.java similarity index 99% rename from java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java rename to java/src/main/java/org/forstdb/MutableColumnFamilyOptionsInterface.java index 729b0e882..59a5c5dfa 100644 --- a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/forstdb/MutableColumnFamilyOptionsInterface.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public interface MutableColumnFamilyOptionsInterface< T extends MutableColumnFamilyOptionsInterface> diff --git a/java/src/main/java/org/rocksdb/MutableDBOptions.java b/java/src/main/java/org/forstdb/MutableDBOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/MutableDBOptions.java rename to java/src/main/java/org/forstdb/MutableDBOptions.java index 927e80522..051303e6c 100644 --- a/java/src/main/java/org/rocksdb/MutableDBOptions.java +++ b/java/src/main/java/org/forstdb/MutableDBOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.HashMap; import java.util.List; diff --git a/java/src/main/java/org/rocksdb/MutableDBOptionsInterface.java b/java/src/main/java/org/forstdb/MutableDBOptionsInterface.java similarity index 99% rename from java/src/main/java/org/rocksdb/MutableDBOptionsInterface.java rename to java/src/main/java/org/forstdb/MutableDBOptionsInterface.java index 8bf7b0d64..f8bb3f5b5 100644 --- a/java/src/main/java/org/rocksdb/MutableDBOptionsInterface.java +++ b/java/src/main/java/org/forstdb/MutableDBOptionsInterface.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; public interface MutableDBOptionsInterface> { /** @@ -418,7 +418,7 @@ public interface MutableDBOptionsInterface * That way RocksDB's compaction is doing sequential instead of random reads. *

    - * Default: 0 + * Default: 2MB * * @param compactionReadaheadSize The compaction read-ahead size * diff --git a/java/src/main/java/org/rocksdb/MutableOptionKey.java b/java/src/main/java/org/forstdb/MutableOptionKey.java similarity index 92% rename from java/src/main/java/org/rocksdb/MutableOptionKey.java rename to java/src/main/java/org/forstdb/MutableOptionKey.java index ec1b9ff3b..315f78c43 100644 --- a/java/src/main/java/org/rocksdb/MutableOptionKey.java +++ b/java/src/main/java/org/forstdb/MutableOptionKey.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; public interface MutableOptionKey { enum ValueType { diff --git a/java/src/main/java/org/rocksdb/MutableOptionValue.java b/java/src/main/java/org/forstdb/MutableOptionValue.java similarity index 99% rename from java/src/main/java/org/rocksdb/MutableOptionValue.java rename to java/src/main/java/org/forstdb/MutableOptionValue.java index fe689b5d0..b2bdca0f1 100644 --- a/java/src/main/java/org/rocksdb/MutableOptionValue.java +++ b/java/src/main/java/org/forstdb/MutableOptionValue.java @@ -1,7 +1,7 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; -import static org.rocksdb.AbstractMutableOptions.INT_ARRAY_INT_SEPARATOR; +import static org.forstdb.AbstractMutableOptions.INT_ARRAY_INT_SEPARATOR; public abstract class MutableOptionValue { diff --git a/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java b/java/src/main/java/org/forstdb/NativeComparatorWrapper.java similarity index 72% rename from java/src/main/java/org/rocksdb/NativeComparatorWrapper.java rename to java/src/main/java/org/forstdb/NativeComparatorWrapper.java index 5ee042a86..1db25332b 100644 --- a/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java +++ b/java/src/main/java/org/forstdb/NativeComparatorWrapper.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; @@ -15,6 +15,8 @@ */ public abstract class NativeComparatorWrapper extends AbstractComparator { + static final String NATIVE_CODE_IMPLEMENTATION_SHOULD_NOT_BE_CALLED = + "This should not be called. Implementation is in Native code"; @Override final ComparatorType getComparatorType() { @@ -23,26 +25,22 @@ final ComparatorType getComparatorType() { @Override public final String name() { - throw new IllegalStateException("This should not be called. " + - "Implementation is in Native code"); + throw new IllegalStateException(NATIVE_CODE_IMPLEMENTATION_SHOULD_NOT_BE_CALLED); } @Override public final int compare(final ByteBuffer s1, final ByteBuffer s2) { - throw new IllegalStateException("This should not be called. " + - "Implementation is in Native code"); + throw new IllegalStateException(NATIVE_CODE_IMPLEMENTATION_SHOULD_NOT_BE_CALLED); } @Override public final void findShortestSeparator(final ByteBuffer start, final ByteBuffer limit) { - throw new IllegalStateException("This should not be called. " + - "Implementation is in Native code"); + throw new IllegalStateException(NATIVE_CODE_IMPLEMENTATION_SHOULD_NOT_BE_CALLED); } @Override public final void findShortSuccessor(final ByteBuffer key) { - throw new IllegalStateException("This should not be called. " + - "Implementation is in Native code"); + throw new IllegalStateException(NATIVE_CODE_IMPLEMENTATION_SHOULD_NOT_BE_CALLED); } /** diff --git a/java/src/main/java/org/rocksdb/NativeLibraryLoader.java b/java/src/main/java/org/forstdb/NativeLibraryLoader.java similarity index 55% rename from java/src/main/java/org/rocksdb/NativeLibraryLoader.java rename to java/src/main/java/org/forstdb/NativeLibraryLoader.java index b97cf28b9..092588fba 100644 --- a/java/src/main/java/org/rocksdb/NativeLibraryLoader.java +++ b/java/src/main/java/org/forstdb/NativeLibraryLoader.java @@ -1,11 +1,11 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import java.io.*; import java.nio.file.Files; import java.nio.file.StandardCopyOption; -import org.rocksdb.util.Environment; +import org.forstdb.util.Environment; /** * This class is used to load the RocksDB shared library from within the jar. @@ -16,14 +16,18 @@ public class NativeLibraryLoader { private static final NativeLibraryLoader instance = new NativeLibraryLoader(); private static boolean initialized = false; - private static final String sharedLibraryName = Environment.getSharedLibraryName("rocksdb"); - private static final String jniLibraryName = Environment.getJniLibraryName("rocksdb"); + private static final String ROCKSDB_LIBRARY_NAME = "forst"; + + private static final String sharedLibraryName = + Environment.getSharedLibraryName(ROCKSDB_LIBRARY_NAME); + private static final String jniLibraryName = Environment.getJniLibraryName(ROCKSDB_LIBRARY_NAME); private static final /* @Nullable */ String fallbackJniLibraryName = - Environment.getFallbackJniLibraryName("rocksdb"); - private static final String jniLibraryFileName = Environment.getJniLibraryFileName("rocksdb"); + Environment.getFallbackJniLibraryName(ROCKSDB_LIBRARY_NAME); + private static final String jniLibraryFileName = + Environment.getJniLibraryFileName(ROCKSDB_LIBRARY_NAME); private static final /* @Nullable */ String fallbackJniLibraryFileName = - Environment.getFallbackJniLibraryFileName("rocksdb"); - private static final String tempFilePrefix = "librocksdbjni"; + Environment.getFallbackJniLibraryFileName(ROCKSDB_LIBRARY_NAME); + private static final String tempFilePrefix = "libforstjni"; private static final String tempFileSuffix = Environment.getJniLibraryExtension(); /** @@ -39,7 +43,7 @@ public static NativeLibraryLoader getInstance() { * Firstly attempts to load the library from java.library.path, * if that fails then it falls back to extracting * the library from the classpath - * {@link org.rocksdb.NativeLibraryLoader#loadLibraryFromJar(java.lang.String)} + * {@link org.forstdb.NativeLibraryLoader#loadLibraryFromJar(java.lang.String)} * * @param tmpDir A temporary directory to use * to copy the native library to when loading from the classpath. @@ -51,6 +55,7 @@ public static NativeLibraryLoader getInstance() { * * @throws java.io.IOException if a filesystem operation fails. */ + @SuppressWarnings("PMD.EmptyCatchBlock") public synchronized void loadLibrary(final String tmpDir) throws IOException { try { // try dynamic library @@ -104,64 +109,58 @@ void loadLibraryFromJar(final String tmpDir) } } - File loadLibraryFromJarToTemp(final String tmpDir) - throws IOException { - InputStream is = null; - try { - // attempt to look up the static library in the jar file - String libraryFileName = jniLibraryFileName; - is = getClass().getClassLoader().getResourceAsStream(libraryFileName); - - if (is == null) { - // is there a fallback we can try - if (fallbackJniLibraryFileName == null) { - throw new RuntimeException(libraryFileName + " was not found inside JAR."); - } - - // attempt to look up the fallback static library in the jar file - libraryFileName = fallbackJniLibraryFileName; - is = getClass().getClassLoader().getResourceAsStream(libraryFileName); - if (is == null) { - throw new RuntimeException(libraryFileName + " was not found inside JAR."); - } + private File createTemp(final String tmpDir, final String libraryFileName) throws IOException { + // create a temporary file to copy the library to + final File temp; + if (tmpDir == null || tmpDir.isEmpty()) { + temp = File.createTempFile(tempFilePrefix, tempFileSuffix); + } else { + final File parentDir = new File(tmpDir); + if (!parentDir.exists()) { + throw new RuntimeException( + "Directory: " + parentDir.getAbsolutePath() + " does not exist!"); } - - // create a temporary file to copy the library to - final File temp; - if (tmpDir == null || tmpDir.isEmpty()) { - temp = File.createTempFile(tempFilePrefix, tempFileSuffix); - } else { - final File parentDir = new File(tmpDir); - if (!parentDir.exists()) { - throw new RuntimeException( - "Directory: " + parentDir.getAbsolutePath() + " does not exist!"); - } - temp = new File(parentDir, libraryFileName); - if (temp.exists() && !temp.delete()) { - throw new RuntimeException( - "File: " + temp.getAbsolutePath() + " already exists and cannot be removed."); - } - if (!temp.createNewFile()) { - throw new RuntimeException("File: " + temp.getAbsolutePath() + " could not be created."); - } + temp = new File(parentDir, libraryFileName); + if (temp.exists() && !temp.delete()) { + throw new RuntimeException( + "File: " + temp.getAbsolutePath() + " already exists and cannot be removed."); } - if (!temp.exists()) { - throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist."); - } else { - temp.deleteOnExit(); + if (!temp.createNewFile()) { + throw new RuntimeException("File: " + temp.getAbsolutePath() + " could not be created."); } + } + if (temp.exists()) { + temp.deleteOnExit(); + return temp; + } else { + throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist."); + } + } - // copy the library from the Jar file to the temp destination - Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + @SuppressWarnings({"PMD.UseProperClassLoader", "PMD.UseTryWithResources"}) + File loadLibraryFromJarToTemp(final String tmpDir) throws IOException { + try (InputStream is = getClass().getClassLoader().getResourceAsStream(jniLibraryFileName)) { + if (is != null) { + final File temp = createTemp(tmpDir, jniLibraryFileName); + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + return temp; + } + } - // return the temporary library file - return temp; + if (fallbackJniLibraryFileName == null) { + throw new RuntimeException(fallbackJniLibraryFileName + " was not found inside JAR."); + } - } finally { + try (InputStream is = + getClass().getClassLoader().getResourceAsStream(fallbackJniLibraryFileName)) { if (is != null) { - is.close(); + final File temp = createTemp(tmpDir, fallbackJniLibraryFileName); + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + return temp; } } + + throw new RuntimeException(jniLibraryFileName + " was not found inside JAR."); } /** diff --git a/java/src/main/java/org/rocksdb/OperationStage.java b/java/src/main/java/org/forstdb/OperationStage.java similarity index 98% rename from java/src/main/java/org/rocksdb/OperationStage.java rename to java/src/main/java/org/forstdb/OperationStage.java index 6ac0a15a2..10c49c8ce 100644 --- a/java/src/main/java/org/rocksdb/OperationStage.java +++ b/java/src/main/java/org/forstdb/OperationStage.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The operation stage. diff --git a/java/src/main/java/org/rocksdb/OperationType.java b/java/src/main/java/org/forstdb/OperationType.java similarity index 98% rename from java/src/main/java/org/rocksdb/OperationType.java rename to java/src/main/java/org/forstdb/OperationType.java index bf7353468..9227427d7 100644 --- a/java/src/main/java/org/rocksdb/OperationType.java +++ b/java/src/main/java/org/forstdb/OperationType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The type used to refer to a thread operation. diff --git a/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java b/java/src/main/java/org/forstdb/OptimisticTransactionDB.java similarity index 96% rename from java/src/main/java/org/rocksdb/OptimisticTransactionDB.java rename to java/src/main/java/org/forstdb/OptimisticTransactionDB.java index ac3cdc210..4ae26d3d9 100644 --- a/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java +++ b/java/src/main/java/org/forstdb/OptimisticTransactionDB.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.List; @@ -27,7 +27,7 @@ private OptimisticTransactionDB(final long nativeHandle) { * Open an OptimisticTransactionDB similar to * {@link RocksDB#open(Options, String)}. * - * @param options {@link org.rocksdb.Options} instance. + * @param options {@link org.forstdb.Options} instance. * @param path the path to the rocksdb. * * @return a {@link OptimisticTransactionDB} instance on success, null if the @@ -52,7 +52,7 @@ public static OptimisticTransactionDB open(final Options options, * Open an OptimisticTransactionDB similar to * {@link RocksDB#open(DBOptions, String, List, List)}. * - * @param dbOptions {@link org.rocksdb.DBOptions} instance. + * @param dbOptions {@link org.forstdb.DBOptions} instance. * @param path the path to the rocksdb. * @param columnFamilyDescriptors list of column family descriptors * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances @@ -86,6 +86,7 @@ public static OptimisticTransactionDB open(final DBOptions dbOptions, // in RocksDB can prevent Java to GC during the life-time of // the currently-created RocksDB. otdb.storeOptionsInstance(dbOptions); + otdb.storeDefaultColumnFamilyHandle(otdb.makeDefaultColumnFamilyHandle()); for (int i = 1; i < handles.length; i++) { columnFamilyHandles.add(new ColumnFamilyHandle(otdb, handles[i])); @@ -107,6 +108,7 @@ public static OptimisticTransactionDB open(final DBOptions dbOptions, * * @throws RocksDBException if an error occurs whilst closing. */ + @Override public void closeE() throws RocksDBException { if (owningHandle_.compareAndSet(true, false)) { try { @@ -128,6 +130,7 @@ public void closeE() throws RocksDBException { *

    * See also {@link #close()}. */ + @SuppressWarnings("PMD.EmptyCatchBlock") @Override public void close() { if (owningHandle_.compareAndSet(true, false)) { diff --git a/java/src/main/java/org/rocksdb/OptimisticTransactionOptions.java b/java/src/main/java/org/forstdb/OptimisticTransactionOptions.java similarity index 98% rename from java/src/main/java/org/rocksdb/OptimisticTransactionOptions.java rename to java/src/main/java/org/forstdb/OptimisticTransactionOptions.java index a2f5d85ab..f1740105d 100644 --- a/java/src/main/java/org/rocksdb/OptimisticTransactionOptions.java +++ b/java/src/main/java/org/forstdb/OptimisticTransactionOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class OptimisticTransactionOptions extends RocksObject implements TransactionalOptions { diff --git a/java/src/main/java/org/rocksdb/OptionString.java b/java/src/main/java/org/forstdb/OptionString.java similarity index 92% rename from java/src/main/java/org/rocksdb/OptionString.java rename to java/src/main/java/org/forstdb/OptionString.java index 61d2a94fe..f26b72cc9 100644 --- a/java/src/main/java/org/rocksdb/OptionString.java +++ b/java/src/main/java/org/forstdb/OptionString.java @@ -3,12 +3,13 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.ArrayList; import java.util.List; import java.util.Objects; +@SuppressWarnings("PMD.AvoidStringBufferField") public class OptionString { private static final char kvPairSeparator = ';'; private static final char kvSeparator = '='; @@ -39,6 +40,7 @@ public static Value fromComplex(final List complex) { return new Value(null, complex); } + @Override public String toString() { final StringBuilder sb = new StringBuilder(); if (isList()) { @@ -68,6 +70,7 @@ private Entry(final String key, final Value value) { this.value = value; } + @Override public String toString() { return "" + key + "=" + value; } @@ -75,6 +78,8 @@ public String toString() { static class Parser { static class Exception extends RuntimeException { + private static final long serialVersionUID = 752283782841276408L; + public Exception(final String s) { super(s); } @@ -122,7 +127,7 @@ private boolean hasNext() { return (sb.length() > 0); } - private boolean is(final char c) { + private boolean isChar(final char c) { return (sb.length() > 0 && sb.charAt(0) == c); } @@ -151,10 +156,10 @@ private String parseKey() { } private String parseSimpleValue() { - if (is(wrappedValueBegin)) { + if (isChar(wrappedValueBegin)) { next(); final String result = parseSimpleValue(); - if (!is(wrappedValueEnd)) { + if (!isChar(wrappedValueEnd)) { exception("Expected to end a wrapped value with " + wrappedValueEnd); } next(); @@ -172,7 +177,7 @@ private List parseList() { final List list = new ArrayList<>(1); while (true) { list.add(parseSimpleValue()); - if (!is(arrayValueSeparator)) + if (!isChar(arrayValueSeparator)) break; next(); @@ -188,7 +193,7 @@ private Entry parseOption() { } final String key = parseKey(); skipWhite(); - if (is(kvSeparator)) { + if (isChar(kvSeparator)) { next(); } else { exception("Expected = separating key and value"); @@ -200,12 +205,12 @@ private Entry parseOption() { private Value parseValue() { skipWhite(); - if (is(complexValueBegin)) { + if (isChar(complexValueBegin)) { next(); skipWhite(); final Value value = Value.fromComplex(parseComplex()); skipWhite(); - if (is(complexValueEnd)) { + if (isChar(complexValueEnd)) { next(); skipWhite(); } else { @@ -214,7 +219,7 @@ private Value parseValue() { return value; } else if (isValueChar()) { return Value.fromList(parseList()); - } else if (is(kvPairSeparator)) { + } else if (isChar(kvPairSeparator)) { // e.g. empty vector embedded in a struct option looks like // struct_opt = {vector_opt=;...} final List entries = new ArrayList<>(); @@ -232,7 +237,7 @@ private List parseComplex() { if (hasNext()) { entries.add(parseOption()); skipWhite(); - while (is(kvPairSeparator)) { + while (isChar(kvPairSeparator)) { next(); skipWhite(); if (!isKeyChar()) { diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/forstdb/Options.java similarity index 98% rename from java/src/main/java/org/rocksdb/Options.java rename to java/src/main/java/org/forstdb/Options.java index 08a07661c..c65978324 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/forstdb/Options.java @@ -3,27 +3,21 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.file.Paths; import java.util.*; /** * Options to control the behavior of a database. It will be used - * during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()). + * during the creation of a {@link org.forstdb.RocksDB} (i.e., RocksDB.open()). *

    * As a descendent of {@link AbstractNativeReference}, this class is {@link AutoCloseable} * and will be automatically released if opened in the preamble of a try with resources block. */ public class Options extends RocksObject - implements DBOptionsInterface, - MutableDBOptionsInterface, - ColumnFamilyOptionsInterface, - MutableColumnFamilyOptionsInterface { - static { - RocksDB.loadLibrary(); - } - + implements DBOptionsInterface, MutableDBOptionsInterface, + ColumnFamilyOptionsInterface, MutableColumnFamilyOptionsInterface { /** * Converts the input properties into a Options-style formatted string * @param properties The set of properties to convert @@ -50,7 +44,7 @@ public static String getOptionStringFromProps(final Properties properties) { * an {@code rocksdb::Options} in the c++ side. */ public Options() { - super(newOptions()); + super(newOptionsInstance()); env_ = Env.getDefault(); } @@ -58,8 +52,8 @@ public Options() { * Construct options for opening a RocksDB. Reusing database options * and column family options. * - * @param dbOptions {@link org.rocksdb.DBOptions} instance - * @param columnFamilyOptions {@link org.rocksdb.ColumnFamilyOptions} + * @param dbOptions {@link org.forstdb.DBOptions} instance + * @param columnFamilyOptions {@link org.forstdb.ColumnFamilyOptions} * instance */ public Options(final DBOptions dbOptions, @@ -840,6 +834,7 @@ public long dbWriteBufferSize() { } @Override + @Deprecated public Options setAccessHintOnCompactionStart(final AccessHint accessHint) { assert(isOwningHandle()); setAccessHintOnCompactionStart(nativeHandle_, accessHint.getValue()); @@ -847,6 +842,7 @@ public Options setAccessHintOnCompactionStart(final AccessHint accessHint) { } @Override + @Deprecated public AccessHint accessHintOnCompactionStart() { assert(isOwningHandle()); return AccessHint.getAccessHint(accessHintOnCompactionStart(nativeHandle_)); @@ -1984,6 +1980,17 @@ public SstPartitionerFactory sstPartitionerFactory() { return sstPartitionerFactory_; } + @Override + public Options setMemtableMaxRangeDeletions(final int count) { + setMemtableMaxRangeDeletions(nativeHandle_, count); + return this; + } + + @Override + public int memtableMaxRangeDeletions() { + return memtableMaxRangeDeletions(nativeHandle_); + } + @Override public Options setCompactionThreadLimiter(final ConcurrentTaskLimiter compactionThreadLimiter) { setCompactionThreadLimiter(nativeHandle_, compactionThreadLimiter.nativeHandle_); @@ -2116,6 +2123,10 @@ public PrepopulateBlobCache prepopulateBlobCache() { // END options for blobs (integrated BlobDB) // + private static long newOptionsInstance() { + RocksDB.loadLibrary(); + return newOptions(); + } private static native long newOptions(); private static native long newOptions(long dbOptHandle, long cfOptHandle); private static native long copyOptions(long handle); @@ -2502,6 +2513,8 @@ private native void setAtomicFlush(final long handle, final boolean atomicFlush); private native boolean atomicFlush(final long handle); private native void setSstPartitionerFactory(long nativeHandle_, long newFactoryHandle); + private native void setMemtableMaxRangeDeletions(final long handle, final int count); + private native int memtableMaxRangeDeletions(final long handle); private static native void setCompactionThreadLimiter( final long nativeHandle_, final long newLimiterHandle); private static native void setAvoidUnnecessaryBlockingIO( @@ -2524,7 +2537,6 @@ private static native void setMaxBgErrorResumeCount( private static native void setBgerrorResumeRetryInterval( final long handle, final long bgerrorResumeRetryInterval); private static native long bgerrorResumeRetryInterval(final long handle); - private native void setEnableBlobFiles(final long nativeHandle_, final boolean enableBlobFiles); private native boolean enableBlobFiles(final long nativeHandle_); private native void setMinBlobSize(final long nativeHandle_, final long minBlobSize); diff --git a/java/src/main/java/org/rocksdb/OptionsUtil.java b/java/src/main/java/org/forstdb/OptionsUtil.java similarity index 78% rename from java/src/main/java/org/rocksdb/OptionsUtil.java rename to java/src/main/java/org/forstdb/OptionsUtil.java index 612023d8e..b41d45049 100644 --- a/java/src/main/java/org/rocksdb/OptionsUtil.java +++ b/java/src/main/java/org/forstdb/OptionsUtil.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.List; @@ -35,10 +35,10 @@ public class OptionsUtil { * BlockBasedTableOptions and making necessary changes. * * @param dbPath the path to the RocksDB. - * @param configOptions {@link org.rocksdb.ConfigOptions} instance. - * @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be + * @param configOptions {@link org.forstdb.ConfigOptions} instance. + * @param dbOptions {@link org.forstdb.DBOptions} instance. This will be * filled and returned. - * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be + * @param cfDescs A list of {@link org.forstdb.ColumnFamilyDescriptor}'s be * returned. * @throws RocksDBException thrown if error happens in underlying * native library. @@ -47,6 +47,7 @@ public static void loadLatestOptions(final ConfigOptions configOptions, final St final DBOptions dbOptions, final List cfDescs) throws RocksDBException { loadLatestOptions(configOptions.nativeHandle_, dbPath, dbOptions.nativeHandle_, cfDescs); + loadTableFormatConfig(cfDescs); } /** @@ -55,10 +56,10 @@ public static void loadLatestOptions(final ConfigOptions configOptions, final St * See LoadLatestOptions above. * * @param optionsFileName the RocksDB options file path. - * @param configOptions {@link org.rocksdb.ConfigOptions} instance. - * @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be + * @param configOptions {@link org.forstdb.ConfigOptions} instance. + * @param dbOptions {@link org.forstdb.DBOptions} instance. This will be * filled and returned. - * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be + * @param cfDescs A list of {@link org.forstdb.ColumnFamilyDescriptor}'s be * returned. * @throws RocksDBException thrown if error happens in underlying * native library. @@ -68,13 +69,14 @@ public static void loadOptionsFromFile(final ConfigOptions configOptions, final List cfDescs) throws RocksDBException { loadOptionsFromFile( configOptions.nativeHandle_, optionsFileName, dbOptions.nativeHandle_, cfDescs); + loadTableFormatConfig(cfDescs); } /** * Returns the latest options file name under the specified RocksDB path. * * @param dbPath the path to the RocksDB. - * @param env {@link org.rocksdb.Env} instance. + * @param env {@link org.forstdb.Env} instance. * @return the latest options file name under the db path. * * @throws RocksDBException thrown if error happens in underlying @@ -85,6 +87,15 @@ public static String getLatestOptionsFileName(final String dbPath, final Env env return getLatestOptionsFileName(dbPath, env.nativeHandle_); } + private static void loadTableFormatConfig(final List cfDescs) { + for (final ColumnFamilyDescriptor columnFamilyDescriptor : cfDescs) { + @SuppressWarnings("PMD.CloseResource") + final ColumnFamilyOptions columnFamilyOptions = columnFamilyDescriptor.getOptions(); + columnFamilyOptions.setFetchedTableFormatConfig( + readTableFormatConfig(columnFamilyOptions.nativeHandle_)); + } + } + /** * Private constructor. * This class has only static methods and shouldn't be instantiated. @@ -98,4 +109,6 @@ private static native void loadOptionsFromFile(long cfgHandle, String optionsFil long dbOptionsHandle, List cfDescs) throws RocksDBException; private static native String getLatestOptionsFileName(String dbPath, long envHandle) throws RocksDBException; + + private native static TableFormatConfig readTableFormatConfig(final long nativeHandle_); } diff --git a/java/src/main/java/org/forstdb/PerfContext.java b/java/src/main/java/org/forstdb/PerfContext.java new file mode 100644 index 000000000..2b7ba6750 --- /dev/null +++ b/java/src/main/java/org/forstdb/PerfContext.java @@ -0,0 +1,761 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +public class PerfContext extends RocksObject { + protected PerfContext(final long nativeHandle) { + super(nativeHandle); + } + + public void reset() { + reset(nativeHandle_); + } + + /** + * @return total number of user key comparisons + */ + public long getUserKeyComparisonCount() { + return getUserKeyComparisonCount(nativeHandle_); + } + + /** + * @return total number of block cache hits + */ + public long getBlockCacheHitCount() { + return getBlockCacheHitCount(nativeHandle_); + } + + /** + * @return total number of block reads (with IO) + */ + public long getBlockReadCount() { + return getBlockReadCount(nativeHandle_); + } + + /** + * @return total number of bytes from block reads + */ + public long getBlockReadByte() { + return getBlockReadByte(nativeHandle_); + } + + /* + @return total nanos spent on block reads + */ + public long getBlockReadTime() { + return getBlockReadTime(nativeHandle_); + } + + /** + * @return total cpu time in nanos spent on block reads + */ + public long getBlockReadCpuTime() { + return getBlockReadCpuTime(nativeHandle_); + } + + /** + * @return total number of index block hits + */ + public long getBlockCacheIndexHitCount() { + return getBlockCacheIndexHitCount(nativeHandle_); + } + + /** + * @return total number of standalone handles lookup from secondary cache + */ + public long getBlockCacheStandaloneHandleCount() { + return getBlockCacheStandaloneHandleCount(nativeHandle_); + } + + /** + * @return total number of real handles lookup from secondary cache that are inserted into + * primary cache + */ + public long getBlockCacheRealHandleCount() { + return getBlockCacheRealHandleCount(nativeHandle_); + } + + /** + * @return total number of index block reads + */ + public long getIndexBlockReadCount() { + return getIndexBlockReadCount(nativeHandle_); + } + + /** + * @return total number of filter block hits + */ + public long getBlockCacheFilterHitCount() { + return getBlockCacheFilterHitCount(nativeHandle_); + } + + /** + * @return total number of filter block reads + */ + public long getFilterBlockReadCount() { + return getFilterBlockReadCount(nativeHandle_); + } + + /** + * @return total number of compression dictionary block reads + */ + public long getCompressionDictBlockReadCount() { + return getCompressionDictBlockReadCount(nativeHandle_); + } + + /** + * @return total number of secondary cache hits + */ + public long getSecondaryCacheHitCount() { + return getSecondaryCacheHitCount(nativeHandle_); + } + + /** + * @return total number of real handles inserted into secondary cache + */ + public long getCompressedSecCacheInsertRealCount() { + return getCompressedSecCacheInsertRealCount(nativeHandle_); + } + + /** + * @return total number of dummy handles inserted into secondary cache + */ + public long getCompressedSecCacheInsertDummyCount() { + return getCompressedSecCacheInsertDummyCount(nativeHandle_); + } + + /** + * @return bytes for vals before compression in secondary cache + */ + public long getCompressedSecCacheUncompressedBytes() { + return getCompressedSecCacheUncompressedBytes(nativeHandle_); + } + + /** + * @return bytes for vals after compression in secondary cache + */ + public long getCompressedSecCacheCompressedBytes() { + return getCompressedSecCacheCompressedBytes(nativeHandle_); + } + + /** + * @return total nanos spent on block checksum + */ + public long getBlockChecksumTime() { + return getBlockChecksumTime(nativeHandle_); + } + + /** + * + * @return total nanos spent on block decompression + */ + public long getBlockDecompressTime() { + return getBlockDecompressTime(nativeHandle_); + } + + /** + * @return bytes for vals returned by Get + */ + public long getReadBytes() { + return getReadBytes(nativeHandle_); + } + + /** + * @return bytes for vals returned by MultiGet + */ + public long getMultigetReadBytes() { + return getMultigetReadBytes(nativeHandle_); + } + + /** + * @return bytes for keys/vals decoded by iterator + */ + public long getIterReadBytes() { + return getIterReadBytes(nativeHandle_); + } + + /** + * @return total number of blob cache hits + */ + public long getBlobCacheHitCount() { + return getBlobCacheHitCount(nativeHandle_); + } + + /** + * @return total number of blob reads (with IO) + */ + public long getBlobReadCount() { + return getBlobReadCount(nativeHandle_); + } + + /** + * @return total number of bytes from blob reads + */ + public long getBlobReadByte() { + return getBlobReadByte(nativeHandle_); + } + + /** + * @return total nanos spent on blob reads + */ + public long getBlobReadTime() { + return getBlobReadTime(nativeHandle_); + } + + /** + * @return total nanos spent on blob checksum + */ + public long getBlobChecksumTime() { + return getBlobChecksumTime(nativeHandle_); + } + + /** + * @return total nanos spent on blob decompression + */ + public long getBlobDecompressTime() { + return getBlobDecompressTime(nativeHandle_); + } + + /** + * total number of internal keys skipped over during iteration. + * There are several reasons for it: + * 1. when calling Next(), the iterator is in the position of the previous + * key, so that we'll need to skip it. It means this counter will always + * be incremented in Next(). + * 2. when calling Next(), we need to skip internal entries for the previous + * keys that are overwritten. + * 3. when calling Next(), Seek() or SeekToFirst(), after previous key + * before calling Next(), the seek key in Seek() or the beginning for + * SeekToFirst(), there may be one or more deleted keys before the next + * valid key that the operation should place the iterator to. We need + * to skip both of the tombstone and updates hidden by the tombstones. The + * tombstones are not included in this counter, while previous updates + * hidden by the tombstones will be included here. + * 4. symmetric cases for Prev() and SeekToLast() + * internal_recent_skipped_count is not included in this counter. + */ + public long getInternalKeySkippedCount() { + return getInternalKeySkippedCount(nativeHandle_); + } + + /** + * Total number of deletes and single deletes skipped over during iteration + * When calling Next(), Seek() or SeekToFirst(), after previous position + * before calling Next(), the seek key in Seek() or the beginning for + * SeekToFirst(), there may be one or more deleted keys before the next valid + * key. Every deleted key is counted once. We don't recount here if there are + * still older updates invalidated by the tombstones. + */ + public long getInternalDeleteSkippedCount() { + return getInternalDeleteSkippedCount(nativeHandle_); + } + + /** + * How many times iterators skipped over internal keys that are more recent + * than the snapshot that iterator is using. + */ + public long getInternalRecentSkippedCount() { + return getInternalRecentSkippedCount(nativeHandle_); + } + + /** + * How many merge operands were fed into the merge operator by iterators. + * Note: base values are not included in the count. + */ + public long getInternalMergeCount() { + return getInternalMergeCount(nativeHandle_); + } + + /** + * How many merge operands were fed into the merge operator by point lookups. + * Note: base values are not included in the count. + */ + public long getInternalMergePointLookupCount() { + return getInternalMergePointLookupCount(nativeHandle_); + } + + /** + * Number of times we reseeked inside a merging iterator, specifically to skip + * after or before a range of keys covered by a range deletion in a newer LSM + * component. + */ + public long getInternalRangeDelReseekCount() { + return getInternalRangeDelReseekCount(nativeHandle_); + } + + /** + * @return total nanos spent on getting snapshot + */ + public long getSnapshotTime() { + return getSnapshotTime(nativeHandle_); + } + + /** + * @return total nanos spent on querying memtables + */ + public long getFromMemtableTime() { + return getFromMemtableTime(nativeHandle_); + } + + /** + * @return number of mem tables queried + */ + public long getFromMemtableCount() { + return getFromMemtableCount(nativeHandle_); + } + + /** + * @return total nanos spent after Get() finds a key + */ + public long getPostProcessTime() { + return getPostProcessTime(nativeHandle_); + } + + /** + * @return total nanos reading from output files + */ + public long getFromOutputFilesTime() { + return getFromOutputFilesTime(nativeHandle_); + } + + /** + * @return total nanos spent on seeking memtable + */ + public long getSeekOnMemtableTime() { + return getSeekOnMemtableTime(nativeHandle_); + } + + /** + * number of seeks issued on memtable + * (including SeekForPrev but not SeekToFirst and SeekToLast) + * @return number of seeks issued on memtable + */ + public long getSeekOnMemtableCount() { + return getSeekOnMemtableCount(nativeHandle_); + } + + /** + * @return number of Next()s issued on memtable + */ + public long getNextOnMemtableCount() { + return getNextOnMemtableCount(nativeHandle_); + } + + /** + * @return number of Prev()s issued on memtable + */ + public long getPrevOnMemtableCount() { + return getPrevOnMemtableCount(nativeHandle_); + } + + /** + * @return total nanos spent on seeking child iters + */ + public long getSeekChildSeekTime() { + return getSeekChildSeekTime(nativeHandle_); + } + + /** + * @return number of seek issued in child iterators + */ + public long getSeekChildSeekCount() { + return getSeekChildSeekCount(nativeHandle_); + } + + /** + * @return total nanos spent on the merge min heap + */ + public long getSeekMinHeapTime() { + return getSeekMinHeapTime(nativeHandle_); + } + + /** + * @return total nanos spent on the merge max heap + */ + public long getSeekMaxHeapTime() { + return getSeekMaxHeapTime(nativeHandle_); + } + + /** + * @return total nanos spent on seeking the internal entries + */ + public long getSeekInternalSeekTime() { + return getSeekInternalSeekTime(nativeHandle_); + } + + /** + * @return total nanos spent on iterating internal entries to find the next user entry + */ + public long getFindNextUserEntryTime() { + return getFindNextUserEntryTime(nativeHandle_); + } + + /** + * @return total nanos spent on writing to WAL + */ + public long getWriteWalTime() { + return getWriteWalTime(nativeHandle_); + } + + /** + * @return total nanos spent on writing to mem tables + */ + public long getWriteMemtableTime() { + return getWriteMemtableTime(nativeHandle_); + } + + /** + * @return total nanos spent on delaying or throttling write + */ + public long getWriteDelayTime() { + return getWriteDelayTime(nativeHandle_); + } + + /** + * @return total nanos spent on switching memtable/wal and scheduling flushes/compactions. + */ + public long getWriteSchedulingFlushesCompactionsTime() { + return getWriteSchedulingFlushesCompactionsTime(nativeHandle_); + } + + /** + * @return total nanos spent on writing a record, excluding the above four things + */ + public long getWritePreAndPostProcessTime() { + return getWritePreAndPostProcessTime(nativeHandle_); + } + + /** + * @return time spent waiting for other threads of the batch group + */ + public long getWriteThreadWaitNanos() { + return getWriteThreadWaitNanos(nativeHandle_); + } + + /** + * @return time spent on acquiring DB mutex. + */ + public long getDbMutexLockNanos() { + return getDbMutexLockNanos(nativeHandle_); + } + + /** + * @return Time spent on waiting with a condition variable created with DB mutex. + */ + public long getDbConditionWaitNanos() { + return getDbConditionWaitNanos(nativeHandle_); + } + + /** + * @return Time spent on merge operator. + */ + public long getMergeOperatorTimeNanos() { + return getMergeOperatorTimeNanos(nativeHandle_); + } + + /** + * @return Time spent on reading index block from block cache or SST file + */ + public long getReadIndexBlockNanos() { + return getReadIndexBlockNanos(nativeHandle_); + } + + /** + * @return Time spent on reading filter block from block cache or SST file + */ + public long getReadFilterBlockNanos() { + return getReadFilterBlockNanos(nativeHandle_); + } + + /** + * @return Time spent on creating data block iterator + */ + public long getNewTableBlockIterNanos() { + return getNewTableBlockIterNanos(nativeHandle_); + } + + /** + * @return Time spent on creating a iterator of an SST file. + */ + public long getNewTableIteratorNanos() { + return getNewTableIteratorNanos(nativeHandle_); + } + + /** + * @return Time spent on seeking a key in data/index blocks + */ + public long getBlockSeekNanos() { + return getBlockSeekNanos(nativeHandle_); + } + /** + * @return Time spent on finding or creating a table reader + */ + public long getFindTableNanos() { + return getFindTableNanos(nativeHandle_); + } + + /** + * @return total number of mem table bloom hits + */ + public long getBloomMemtableHitCount() { + return getBloomMemtableHitCount(nativeHandle_); + } + + // total number of mem table bloom misses + public long getBloomMemtableMissCount() { + return getBloomMemtableMissCount(nativeHandle_); + } + + /** + * @return total number of SST bloom hits + */ + public long getBloomSstHitCount() { + return getBloomSstHitCount(nativeHandle_); + } + + /** + * @return total number of SST bloom misses + */ + public long getBloomSstMissCount() { + return getBloomSstMissCount(nativeHandle_); + } + + /** + * @return Time spent waiting on key locks in transaction lock manager. + */ + public long getKeyLockWaitTime() { + return getKeyLockWaitTime(nativeHandle_); + } + /** + * @return number of times acquiring a lock was blocked by another transaction. + */ + public long getKeyLockWaitCount() { + return getKeyLockWaitCount(nativeHandle_); + } + + /** + * @return Total time spent in Env filesystem operations. These are only populated when TimedEnv + * is used. + */ + public long getEnvNewSequentialFileNanos() { + return getEnvNewSequentialFileNanos(nativeHandle_); + } + + public long getEnvNewRandomAccessFileNanos() { + return getEnvNewRandomAccessFileNanos(nativeHandle_); + } + + public long getEnvNewWritableFileNanos() { + return getEnvNewWritableFileNanos(nativeHandle_); + } + + public long getEnvReuseWritableFileNanos() { + return getEnvReuseWritableFileNanos(nativeHandle_); + } + + public long getEnvNewRandomRwFileNanos() { + return getEnvNewRandomRwFileNanos(nativeHandle_); + } + + public long getEnvNewDirectoryNanos() { + return getEnvNewDirectoryNanos(nativeHandle_); + } + + public long getEnvFileExistsNanos() { + return getEnvFileExistsNanos(nativeHandle_); + } + public long getEnvGetChildrenNanos() { + return getEnvGetChildrenNanos(nativeHandle_); + } + + public long getEnvGetChildrenFileAttributesNanos() { + return getEnvGetChildrenFileAttributesNanos(nativeHandle_); + } + + public long getEnvDeleteFileNanos() { + return getEnvDeleteFileNanos(nativeHandle_); + } + + public long getEnvCreateDirNanos() { + return getEnvCreateDirNanos(nativeHandle_); + } + public long getEnvCreateDirIfMissingNanos() { + return getEnvCreateDirIfMissingNanos(nativeHandle_); + } + + public long getEnvDeleteDirNanos() { + return getEnvDeleteDirNanos(nativeHandle_); + } + + public long getEnvGetFileSizeNanos() { + return getEnvGetFileSizeNanos(nativeHandle_); + } + + public long getEnvGetFileModificationTimeNanos() { + return getEnvGetFileModificationTimeNanos(nativeHandle_); + } + + public long getEnvRenameFileNanos() { + return getEnvRenameFileNanos(nativeHandle_); + } + + public long getEnvLinkFileNanos() { + return getEnvLinkFileNanos(nativeHandle_); + } + + public long getEnvLockFileNanos() { + return getEnvLockFileNanos(nativeHandle_); + } + + public long getEnvUnlockFileNanos() { + return getEnvUnlockFileNanos(nativeHandle_); + } + + public long getEnvNewLoggerNanos() { + return getEnvNewLoggerNanos(nativeHandle_); + } + + public long getGetCpuNanos() { + return getGetCpuNanos(nativeHandle_); + } + + public long getIterNextCpuNanos() { + return getIterNextCpuNanos(nativeHandle_); + } + public long getIterPrevCpuNanos() { + return getIterPrevCpuNanos(nativeHandle_); + } + + public long getIterSeekCpuNanos() { + return getIterSeekCpuNanos(nativeHandle_); + } + + /** + * @return Time spent in encrypting data. Populated when EncryptedEnv is used. + */ + public long getEncryptDataNanos() { + return getEncryptDataNanos(nativeHandle_); + } + + /** + * @return Time spent in decrypting data. Populated when EncryptedEnv is used. + */ + public long getDecryptDataNanos() { + return getDecryptDataNanos(nativeHandle_); + } + + public long getNumberAsyncSeek() { + return getNumberAsyncSeek(nativeHandle_); + } + + @Override + protected void disposeInternal(long handle) { + // Nothing to do. Perf context is valid for all the time of application is running. + } + + private native void reset(final long nativeHandle); + + private native long getUserKeyComparisonCount(final long handle); + private native long getBlockCacheHitCount(final long handle); + private native long getBlockReadCount(final long handle); + private native long getBlockReadByte(final long handle); + private native long getBlockReadTime(final long handle); + private native long getBlockReadCpuTime(final long handle); + private native long getBlockCacheIndexHitCount(final long handle); + private native long getBlockCacheStandaloneHandleCount(final long handle); + private native long getBlockCacheRealHandleCount(final long handle); + private native long getIndexBlockReadCount(final long handle); + private native long getBlockCacheFilterHitCount(final long handle); + private native long getFilterBlockReadCount(final long handle); + private native long getCompressionDictBlockReadCount(final long handle); + + private native long getSecondaryCacheHitCount(long handle); + private native long getCompressedSecCacheInsertRealCount(long handle); + + private native long getCompressedSecCacheInsertDummyCount(final long handle); + private native long getCompressedSecCacheUncompressedBytes(final long handle); + private native long getCompressedSecCacheCompressedBytes(final long handle); + private native long getBlockChecksumTime(final long handle); + private native long getBlockDecompressTime(final long handle); + private native long getReadBytes(final long handle); + private native long getMultigetReadBytes(final long handle); + private native long getIterReadBytes(final long handle); + private native long getBlobCacheHitCount(final long handle); + private native long getBlobReadCount(final long handle); + private native long getBlobReadByte(final long handle); + private native long getBlobReadTime(final long handle); + private native long getBlobChecksumTime(final long handle); + private native long getBlobDecompressTime(final long handle); + private native long getInternalKeySkippedCount(final long handle); + private native long getInternalDeleteSkippedCount(final long handle); + private native long getInternalRecentSkippedCount(final long handle); + private native long getInternalMergeCount(final long handle); + private native long getInternalMergePointLookupCount(final long handle); + private native long getInternalRangeDelReseekCount(final long handle); + private native long getSnapshotTime(final long handle); + private native long getFromMemtableTime(final long handle); + private native long getFromMemtableCount(final long handle); + private native long getPostProcessTime(final long handle); + private native long getFromOutputFilesTime(final long handle); + private native long getSeekOnMemtableTime(final long handle); + private native long getSeekOnMemtableCount(final long handle); + private native long getNextOnMemtableCount(final long handle); + private native long getPrevOnMemtableCount(final long handle); + private native long getSeekChildSeekTime(final long handle); + private native long getSeekChildSeekCount(final long handle); + private native long getSeekMinHeapTime(final long handle); + private native long getSeekMaxHeapTime(final long handle); + private native long getSeekInternalSeekTime(final long handle); + private native long getFindNextUserEntryTime(final long handle); + private native long getWriteWalTime(long handle); + private native long getWriteMemtableTime(long handle); + private native long getWriteDelayTime(long handle); + private native long getWriteSchedulingFlushesCompactionsTime(long handle); + private native long getWritePreAndPostProcessTime(long handle); + private native long getWriteThreadWaitNanos(long handle); + private native long getDbMutexLockNanos(long handle); + private native long getDbConditionWaitNanos(long handle); + private native long getMergeOperatorTimeNanos(long handle); + private native long getReadIndexBlockNanos(long handle); + private native long getReadFilterBlockNanos(long handle); + private native long getNewTableBlockIterNanos(long handle); + private native long getNewTableIteratorNanos(long handle); + private native long getBlockSeekNanos(long handle); + private native long getFindTableNanos(long handle); + private native long getBloomMemtableHitCount(long handle); + private native long getBloomMemtableMissCount(long handle); + private native long getBloomSstHitCount(long handle); + private native long getBloomSstMissCount(long handle); + private native long getKeyLockWaitTime(long handle); + private native long getKeyLockWaitCount(long handle); + private native long getEnvNewSequentialFileNanos(long handle); + private native long getEnvNewRandomAccessFileNanos(long handle); + private native long getEnvNewWritableFileNanos(long handle); + private native long getEnvReuseWritableFileNanos(long handle); + private native long getEnvNewRandomRwFileNanos(long handle); + private native long getEnvNewDirectoryNanos(long handle); + private native long getEnvFileExistsNanos(long handle); + private native long getEnvGetChildrenNanos(long handle); + private native long getEnvGetChildrenFileAttributesNanos(long handle); + private native long getEnvDeleteFileNanos(long handle); + private native long getEnvCreateDirNanos(long handle); + private native long getEnvCreateDirIfMissingNanos(long handle); + private native long getEnvDeleteDirNanos(long handle); + private native long getEnvGetFileSizeNanos(long handle); + private native long getEnvGetFileModificationTimeNanos(long handle); + private native long getEnvRenameFileNanos(long handle); + private native long getEnvLinkFileNanos(long handle); + private native long getEnvLockFileNanos(long handle); + private native long getEnvUnlockFileNanos(long handle); + private native long getEnvNewLoggerNanos(long handle); + private native long getGetCpuNanos(long nativeHandle_); + private native long getIterNextCpuNanos(long nativeHandle_); + private native long getIterPrevCpuNanos(long nativeHandle_); + private native long getIterSeekCpuNanos(long nativeHandle_); + private native long getEncryptDataNanos(long nativeHandle_); + private native long getDecryptDataNanos(long nativeHandle_); + private native long getNumberAsyncSeek(long nativeHandle_); +} diff --git a/java/src/main/java/org/forstdb/PerfLevel.java b/java/src/main/java/org/forstdb/PerfLevel.java new file mode 100644 index 000000000..2d90366e6 --- /dev/null +++ b/java/src/main/java/org/forstdb/PerfLevel.java @@ -0,0 +1,60 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +public enum PerfLevel { + /** + * Unknown setting + */ + UNINITIALIZED((byte) 0), + /** + * disable perf stats + */ + DISABLE((byte) 1), + /** + * enable only count stats + */ + ENABLE_COUNT((byte) 2), + /** + * Other than count stats, also enable time stats except for mutexes + */ + ENABLE_TIME_EXCEPT_FOR_MUTEX((byte) 3), + + /** + * Other than time, also measure CPU time counters. Still don't measure + * time (neither wall time nor CPU time) for mutexes + */ + ENABLE_TIME_AND_CPU_TIME_EXCEPT_FOR_MUTEX((byte) 4), + /** + * enable count and time stats + */ + ENABLE_TIME((byte) 5), + + /** + * Do not use + * @deprecated It's here to just keep parity with C++ API. + */ + @Deprecated OUT_OF_BOUNDS((byte) 6); + + PerfLevel(byte _value) { + this._value = _value; + } + + private final byte _value; + + public byte getValue() { + return _value; + } + + public static PerfLevel getPerfLevel(byte level) { + for (PerfLevel l : PerfLevel.values()) { + if (l.getValue() == level) { + return l; + } + } + throw new IllegalArgumentException("Uknknown PerfLevel constant : " + level); + } +} diff --git a/java/src/main/java/org/rocksdb/PersistentCache.java b/java/src/main/java/org/forstdb/PersistentCache.java similarity index 97% rename from java/src/main/java/org/rocksdb/PersistentCache.java rename to java/src/main/java/org/forstdb/PersistentCache.java index 5297111e6..a349a9461 100644 --- a/java/src/main/java/org/rocksdb/PersistentCache.java +++ b/java/src/main/java/org/forstdb/PersistentCache.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Persistent cache for caching IO pages on a persistent medium. The diff --git a/java/src/main/java/org/rocksdb/PlainTableConfig.java b/java/src/main/java/org/forstdb/PlainTableConfig.java similarity index 98% rename from java/src/main/java/org/rocksdb/PlainTableConfig.java rename to java/src/main/java/org/forstdb/PlainTableConfig.java index 46077ba56..6c62e589a 100644 --- a/java/src/main/java/org/rocksdb/PlainTableConfig.java +++ b/java/src/main/java/org/forstdb/PlainTableConfig.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The config for plain table sst format. @@ -163,7 +163,7 @@ public int hugePageTlbSize() { * different encoding types can co-exist in the same DB and * can be read.

    * - * @param encodingType {@link org.rocksdb.EncodingType} value. + * @param encodingType {@link org.forstdb.EncodingType} value. * @return the reference to the current config. */ public PlainTableConfig setEncodingType(final EncodingType encodingType) { diff --git a/java/src/main/java/org/rocksdb/PrepopulateBlobCache.java b/java/src/main/java/org/forstdb/PrepopulateBlobCache.java similarity index 99% rename from java/src/main/java/org/rocksdb/PrepopulateBlobCache.java rename to java/src/main/java/org/forstdb/PrepopulateBlobCache.java index f1237aa7c..0a4329289 100644 --- a/java/src/main/java/org/rocksdb/PrepopulateBlobCache.java +++ b/java/src/main/java/org/forstdb/PrepopulateBlobCache.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Enum PrepopulateBlobCache diff --git a/java/src/main/java/org/rocksdb/Priority.java b/java/src/main/java/org/forstdb/Priority.java similarity index 93% rename from java/src/main/java/org/rocksdb/Priority.java rename to java/src/main/java/org/forstdb/Priority.java index 34a56edcb..2077739e8 100644 --- a/java/src/main/java/org/rocksdb/Priority.java +++ b/java/src/main/java/org/forstdb/Priority.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The Thread Pool priority. @@ -34,7 +34,7 @@ byte getValue() { * * @param value byte representation of Priority. * - * @return {@link org.rocksdb.Priority} instance. + * @return {@link org.forstdb.Priority} instance. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/Range.java b/java/src/main/java/org/forstdb/Range.java similarity index 95% rename from java/src/main/java/org/rocksdb/Range.java rename to java/src/main/java/org/forstdb/Range.java index 74c85e5f0..48f32a1ad 100644 --- a/java/src/main/java/org/rocksdb/Range.java +++ b/java/src/main/java/org/forstdb/Range.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Range from start to limit. diff --git a/java/src/main/java/org/rocksdb/RateLimiter.java b/java/src/main/java/org/forstdb/RateLimiter.java similarity index 99% rename from java/src/main/java/org/rocksdb/RateLimiter.java rename to java/src/main/java/org/forstdb/RateLimiter.java index c2b8a0fd9..ace2b9500 100644 --- a/java/src/main/java/org/rocksdb/RateLimiter.java +++ b/java/src/main/java/org/forstdb/RateLimiter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * RateLimiter, which is used to control write rate of flush and diff --git a/java/src/main/java/org/rocksdb/RateLimiterMode.java b/java/src/main/java/org/forstdb/RateLimiterMode.java similarity index 98% rename from java/src/main/java/org/rocksdb/RateLimiterMode.java rename to java/src/main/java/org/forstdb/RateLimiterMode.java index 4b029d816..87615fa10 100644 --- a/java/src/main/java/org/rocksdb/RateLimiterMode.java +++ b/java/src/main/java/org/forstdb/RateLimiterMode.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Mode for {@link RateLimiter#RateLimiter(long, long, int, RateLimiterMode)}. diff --git a/java/src/main/java/org/rocksdb/ReadOptions.java b/java/src/main/java/org/forstdb/ReadOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/ReadOptions.java rename to java/src/main/java/org/forstdb/ReadOptions.java index 65b781d16..5da35981c 100644 --- a/java/src/main/java/org/rocksdb/ReadOptions.java +++ b/java/src/main/java/org/forstdb/ReadOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The class that controls the get behavior. @@ -576,11 +576,10 @@ public ReadOptions setAutoPrefixMode(final boolean mode) { public Slice timestamp() { assert (isOwningHandle()); final long timestampSliceHandle = timestamp(nativeHandle_); - if (timestampSliceHandle != 0) { - return new Slice(timestampSliceHandle); - } else { + if (timestampSliceHandle == 0) { return null; } + return new Slice(timestampSliceHandle); } /** @@ -626,11 +625,10 @@ public ReadOptions setTimestamp(final AbstractSlice timestamp) { public Slice iterStartTs() { assert (isOwningHandle()); final long iterStartTsHandle = iterStartTs(nativeHandle_); - if (iterStartTsHandle != 0) { - return new Slice(iterStartTsHandle); - } else { + if (iterStartTsHandle == 0) { return null; } + return new Slice(iterStartTsHandle); } /** diff --git a/java/src/main/java/org/rocksdb/ReadTier.java b/java/src/main/java/org/forstdb/ReadTier.java similarity index 93% rename from java/src/main/java/org/rocksdb/ReadTier.java rename to java/src/main/java/org/forstdb/ReadTier.java index 78f83f6ad..7970918cd 100644 --- a/java/src/main/java/org/rocksdb/ReadTier.java +++ b/java/src/main/java/org/forstdb/ReadTier.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * RocksDB {@link ReadOptions} read tiers. @@ -34,7 +34,7 @@ public byte getValue() { * * @param value byte representation of ReadTier. * - * @return {@link org.rocksdb.ReadTier} instance or null. + * @return {@link org.forstdb.ReadTier} instance or null. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/RemoveEmptyValueCompactionFilter.java b/java/src/main/java/org/forstdb/RemoveEmptyValueCompactionFilter.java similarity index 96% rename from java/src/main/java/org/rocksdb/RemoveEmptyValueCompactionFilter.java rename to java/src/main/java/org/forstdb/RemoveEmptyValueCompactionFilter.java index e96694313..20e187205 100644 --- a/java/src/main/java/org/rocksdb/RemoveEmptyValueCompactionFilter.java +++ b/java/src/main/java/org/forstdb/RemoveEmptyValueCompactionFilter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Just a Java wrapper around EmptyValueCompactionFilter implemented in C++ diff --git a/java/src/main/java/org/rocksdb/RestoreOptions.java b/java/src/main/java/org/forstdb/RestoreOptions.java similarity index 98% rename from java/src/main/java/org/rocksdb/RestoreOptions.java rename to java/src/main/java/org/forstdb/RestoreOptions.java index a6b43d476..6c60981a6 100644 --- a/java/src/main/java/org/rocksdb/RestoreOptions.java +++ b/java/src/main/java/org/forstdb/RestoreOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * RestoreOptions to control the behavior of restore. diff --git a/java/src/main/java/org/rocksdb/ReusedSynchronisationType.java b/java/src/main/java/org/forstdb/ReusedSynchronisationType.java similarity index 95% rename from java/src/main/java/org/rocksdb/ReusedSynchronisationType.java rename to java/src/main/java/org/forstdb/ReusedSynchronisationType.java index 2709a5d59..a3590fabc 100644 --- a/java/src/main/java/org/rocksdb/ReusedSynchronisationType.java +++ b/java/src/main/java/org/forstdb/ReusedSynchronisationType.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Determines the type of synchronisation primitive used @@ -47,7 +47,7 @@ public byte getValue() { * * @param value byte representation of ReusedSynchronisationType. * - * @return {@link org.rocksdb.ReusedSynchronisationType} instance. + * @return {@link org.forstdb.ReusedSynchronisationType} instance. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/RocksCallbackObject.java b/java/src/main/java/org/forstdb/RocksCallbackObject.java similarity index 97% rename from java/src/main/java/org/rocksdb/RocksCallbackObject.java rename to java/src/main/java/org/forstdb/RocksCallbackObject.java index 2b9de4b8e..4daf14277 100644 --- a/java/src/main/java/org/rocksdb/RocksCallbackObject.java +++ b/java/src/main/java/org/forstdb/RocksCallbackObject.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.List; @@ -39,7 +39,7 @@ protected RocksCallbackObject(final long... nativeParameterHandles) { static /* @Nullable */ long[] toNativeHandleList( /* @Nullable */ final List objectList) { if (objectList == null) { - return null; + return new long[0]; } final int len = objectList.size(); final long[] handleList = new long[len]; diff --git a/java/src/main/java/org/rocksdb/RocksDB.java b/java/src/main/java/org/forstdb/RocksDB.java similarity index 86% rename from java/src/main/java/org/rocksdb/RocksDB.java rename to java/src/main/java/org/forstdb/RocksDB.java index f46cf6846..ede573454 100644 --- a/java/src/main/java/org/rocksdb/RocksDB.java +++ b/java/src/main/java/org/forstdb/RocksDB.java @@ -3,15 +3,16 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.forstdb.util.BufferUtil.CheckBounds; import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.atomic.AtomicReference; -import org.rocksdb.util.Environment; +import org.forstdb.util.Environment; /** * A RocksDB is a persistent ordered map from keys to values. It is safe for @@ -32,9 +33,13 @@ private enum LibraryState { private static final AtomicReference libraryLoaded = new AtomicReference<>(LibraryState.NOT_LOADED); - static { - RocksDB.loadLibrary(); - } + static final String PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD = + "Performance optimization for a very specific workload"; + + private static final String BB_ALL_DIRECT_OR_INDIRECT = + "ByteBuffer parameters must all be direct, or must all be indirect"; + private ColumnFamilyHandle defaultColumnFamilyHandle_; + private final ReadOptions defaultReadOptions_ = new ReadOptions(); private final List ownedColumnFamilyHandles = new ArrayList<>(); @@ -45,6 +50,7 @@ private enum LibraryState { * java.io.tmpdir, however, you can override this temporary location by * setting the environment variable ROCKSDB_SHAREDLIB_DIR. */ + @SuppressWarnings("PMD.EmptyCatchBlock") public static void loadLibrary() { if (libraryLoaded.get() == LibraryState.LOADED) { return; @@ -82,7 +88,8 @@ public static void loadLibrary() { try { Thread.sleep(10); } catch(final InterruptedException e) { - //ignore + throw new RuntimeException("Loading the RocksDB shared library is interrupted", + e); } } } @@ -94,6 +101,7 @@ public static void loadLibrary() { * @param paths a list of strings where each describes a directory * of a library. */ + @SuppressWarnings("PMD.EmptyCatchBlock") public static void loadLibrary(final List paths) { if (libraryLoaded.get() == LibraryState.LOADED) { return; @@ -143,7 +151,8 @@ public static void loadLibrary(final List paths) { try { Thread.sleep(10); } catch(final InterruptedException e) { - //ignore + throw new RuntimeException("Loading the RocksDB shared library is interrupted", + e); } } } @@ -175,9 +184,11 @@ protected RocksDB(final long nativeHandle) { * @see Options#setCreateIfMissing(boolean) */ public static RocksDB open(final String path) throws RocksDBException { - final Options options = new Options(); - options.setCreateIfMissing(true); - return open(options, path); + RocksDB.loadLibrary(); + try (Options options = new Options()) { + options.setCreateIfMissing(true); + return open(options, path); + } } /** @@ -213,8 +224,9 @@ public static RocksDB open(final String path, final List columnFamilyDescriptors, final List columnFamilyHandles) throws RocksDBException { - final DBOptions options = new DBOptions(); - return open(options, path, columnFamilyDescriptors, columnFamilyHandles); + try (DBOptions options = new DBOptions()) { + return open(options, path, columnFamilyDescriptors, columnFamilyHandles); + } } /** @@ -231,7 +243,7 @@ public static RocksDB open(final String path, * with new Options instance as underlying native statistics instance does not * use any locks to prevent concurrent updates.

    * - * @param options {@link org.rocksdb.Options} instance. + * @param options {@link org.forstdb.Options} instance. * @param path the path to the rocksdb. * @return a {@link RocksDB} instance on success, null if the specified * {@link RocksDB} can not be opened. @@ -248,6 +260,7 @@ public static RocksDB open(final Options options, final String path) // the currently-created RocksDB. final RocksDB db = new RocksDB(open(options.nativeHandle_, path)); db.storeOptionsInstance(options); + db.storeDefaultColumnFamilyHandle(db.makeDefaultColumnFamilyHandle()); return db; } @@ -274,7 +287,7 @@ public static RocksDB open(final Options options, final String path) * ColumnFamily handles are disposed when the RocksDB instance is disposed. *

    * - * @param options {@link org.rocksdb.DBOptions} instance. + * @param options {@link org.forstdb.DBOptions} instance. * @param path the path to the rocksdb. * @param columnFamilyDescriptors list of column family descriptors * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances @@ -307,11 +320,13 @@ public static RocksDB open(final DBOptions options, final String path, db.storeOptionsInstance(options); for (int i = 1; i < handles.length; i++) { - final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(db, handles[i]); + final ColumnFamilyHandle columnFamilyHandle = // NOPMD - CloseResource + new ColumnFamilyHandle(db, handles[i]); columnFamilyHandles.add(columnFamilyHandle); } db.ownedColumnFamilyHandles.addAll(columnFamilyHandles); + db.storeDefaultColumnFamilyHandle(db.makeDefaultColumnFamilyHandle()); return db; } @@ -330,10 +345,12 @@ public static RocksDB open(final DBOptions options, final String path, */ public static RocksDB openReadOnly(final String path) throws RocksDBException { + RocksDB.loadLibrary(); // This allows to use the rocksjni default Options instead of // the c++ one. - final Options options = new Options(); - return openReadOnly(options, path); + try (Options options = new Options()) { + return openReadOnly(options, path); + } } /** @@ -384,6 +401,7 @@ public static RocksDB openReadOnly(final Options options, final String path, // the currently-created RocksDB. final RocksDB db = new RocksDB(openROnly(options.nativeHandle_, path, errorIfWalFileExists)); db.storeOptionsInstance(options); + db.storeDefaultColumnFamilyHandle(db.makeDefaultColumnFamilyHandle()); return db; } @@ -408,8 +426,9 @@ public static RocksDB openReadOnly(final String path, throws RocksDBException { // This allows to use the rocksjni default Options instead of // the c++ one. - final DBOptions options = new DBOptions(); - return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles, false); + try (DBOptions options = new DBOptions()) { + return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles, false); + } } /** @@ -487,11 +506,13 @@ public static RocksDB openReadOnly(final DBOptions options, final String path, db.storeOptionsInstance(options); for (int i = 1; i < handles.length; i++) { - final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(db, handles[i]); + final ColumnFamilyHandle columnFamilyHandle = // NOPMD - CloseResource + new ColumnFamilyHandle(db, handles[i]); columnFamilyHandles.add(columnFamilyHandle); } db.ownedColumnFamilyHandles.addAll(columnFamilyHandles); + db.storeDefaultColumnFamilyHandle(db.makeDefaultColumnFamilyHandle()); return db; } @@ -529,6 +550,7 @@ public static RocksDB openAsSecondary(final Options options, final String path, // the currently-created RocksDB. final RocksDB db = new RocksDB(openAsSecondary(options.nativeHandle_, path, secondaryPath)); db.storeOptionsInstance(options); + db.storeDefaultColumnFamilyHandle(db.makeDefaultColumnFamilyHandle()); return db; } @@ -583,11 +605,13 @@ public static RocksDB openAsSecondary(final DBOptions options, final String path db.storeOptionsInstance(options); for (int i = 1; i < handles.length; i++) { - final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(db, handles[i]); + final ColumnFamilyHandle columnFamilyHandle = // NOPMD - CloseResource + new ColumnFamilyHandle(db, handles[i]); columnFamilyHandles.add(columnFamilyHandle); } db.ownedColumnFamilyHandles.addAll(columnFamilyHandles); + db.storeDefaultColumnFamilyHandle(db.makeDefaultColumnFamilyHandle()); return db; } @@ -606,7 +630,8 @@ public static RocksDB openAsSecondary(final DBOptions options, final String path * @throws RocksDBException if an error occurs whilst closing. */ public void closeE() throws RocksDBException { - for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) { + for (final ColumnFamilyHandle columnFamilyHandle : // NOPMD - CloseResource + ownedColumnFamilyHandles) { columnFamilyHandle.close(); } ownedColumnFamilyHandles.clear(); @@ -631,9 +656,11 @@ public void closeE() throws RocksDBException { *

    * See also {@link #close()}. */ + @SuppressWarnings("PMD.EmptyCatchBlock") @Override public void close() { - for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) { + for (final ColumnFamilyHandle columnFamilyHandle : // NOPMD - CloseResource + ownedColumnFamilyHandles) { columnFamilyHandle.close(); } ownedColumnFamilyHandles.clear(); @@ -672,7 +699,7 @@ public static List listColumnFamilies(final Options options, * The ColumnFamilyHandle is automatically disposed with DB disposal. * * @param columnFamilyDescriptor column family to be created. - * @return {@link org.rocksdb.ColumnFamilyHandle} instance. + * @return {@link org.forstdb.ColumnFamilyHandle} instance. * * @throws RocksDBException thrown if error happens in underlying * native library. @@ -709,7 +736,7 @@ public List createColumnFamilies( final List columnFamilyHandles = new ArrayList<>(cfHandles.length); for (final long cfHandle : cfHandles) { - final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(this, cfHandle); + final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(this, cfHandle); // NOPMD columnFamilyHandles.add(columnFamilyHandle); } ownedColumnFamilyHandles.addAll(columnFamilyHandles); @@ -742,7 +769,7 @@ public List createColumnFamilies( final List columnFamilyHandles = new ArrayList<>(cfHandles.length); for (final long cfHandle : cfHandles) { - final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(this, cfHandle); + final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(this, cfHandle); // NOPMD columnFamilyHandles.add(columnFamilyHandle); } ownedColumnFamilyHandles.addAll(columnFamilyHandles); @@ -756,19 +783,7 @@ public List createColumnFamilies( * The ColumnFamilyHandle is automatically disposed with DB disposal. * * @param columnFamilyDescriptor column family to be created. - * @return {@link org.rocksdb.ColumnFamilyHandle} instance. - * - * @throws RocksDBException thrown if error happens in underlying - * native library. - */ - /** - * Creates a new column family with the name columnFamilyName and - * import external SST files specified in `metadata` allocates a - * ColumnFamilyHandle within an internal structure. - * The ColumnFamilyHandle is automatically disposed with DB disposal. - * - * @param columnFamilyDescriptor column family to be created. - * @return {@link org.rocksdb.ColumnFamilyHandle} instance. + * @return {@link org.forstdb.ColumnFamilyHandle} instance. * * @throws RocksDBException thrown if error happens in underlying * native library. @@ -788,15 +803,15 @@ public ColumnFamilyHandle createColumnFamilyWithImport( final ImportColumnFamilyOptions importColumnFamilyOptions, final List metadatas) throws RocksDBException { final int metadataNum = metadatas.size(); - final long[] metadataHandeList = new long[metadataNum]; + final long[] metadataHandleList = new long[metadataNum]; for (int i = 0; i < metadataNum; i++) { - metadataHandeList[i] = metadatas.get(i).newExportImportFilesMetaDataHandle(); + metadataHandleList[i] = metadatas.get(i).getNativeHandle(); } final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(this, createColumnFamilyWithImport(nativeHandle_, columnFamilyDescriptor.getName(), columnFamilyDescriptor.getName().length, columnFamilyDescriptor.getOptions().nativeHandle_, - importColumnFamilyOptions.nativeHandle_, metadataHandeList)); + importColumnFamilyOptions.nativeHandle_, metadataHandleList)); ownedColumnFamilyHandles.add(columnFamilyHandle); return columnFamilyHandle; } @@ -806,7 +821,7 @@ public ColumnFamilyHandle createColumnFamilyWithImport( * only records a drop record in the manifest and prevents the column * family from flushing and compacting. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * * @throws RocksDBException thrown if error happens in underlying @@ -838,7 +853,7 @@ public void dropColumnFamilies( */ public void destroyColumnFamilyHandle(final ColumnFamilyHandle columnFamilyHandle) { for (int i = 0; i < ownedColumnFamilyHandles.size(); ++i) { - final ColumnFamilyHandle ownedHandle = ownedColumnFamilyHandles.get(i); + final ColumnFamilyHandle ownedHandle = ownedColumnFamilyHandles.get(i); // NOPMD if (ownedHandle.equals(columnFamilyHandle)) { columnFamilyHandle.close(); ownedColumnFamilyHandles.remove(i); @@ -882,8 +897,8 @@ public void put(final byte[] key, final byte[] value) public void put(final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); put(nativeHandle_, key, offset, len, value, vOffset, vLen); } @@ -891,7 +906,7 @@ public void put(final byte[] key, final int offset, final int len, * Set the database entry for "key" to "value" in the specified * column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the specified key to be inserted. * @param value the value associated with the specified key. @@ -911,7 +926,7 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, * Set the database entry for "key" to "value" in the specified * column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key The specified key to be inserted * @param offset the offset of the "key" array to be used, must @@ -932,8 +947,8 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); put(nativeHandle_, key, offset, len, value, vOffset, vLen, columnFamilyHandle.nativeHandle_); } @@ -941,7 +956,7 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, /** * Set the database entry for "key" to "value". * - * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param writeOpts {@link org.forstdb.WriteOptions} instance. * @param key the specified key to be inserted. * @param value the value associated with the specified key. * @@ -957,7 +972,7 @@ public void put(final WriteOptions writeOpts, final byte[] key, /** * Set the database entry for "key" to "value". * - * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param writeOpts {@link org.forstdb.WriteOptions} instance. * @param key The specified key to be inserted * @param offset the offset of the "key" array to be used, must be * non-negative and no larger than "key".length @@ -977,8 +992,8 @@ public void put(final WriteOptions writeOpts, final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); put(nativeHandle_, writeOpts.nativeHandle_, key, offset, len, value, vOffset, vLen); } @@ -987,9 +1002,9 @@ public void put(final WriteOptions writeOpts, * Set the database entry for "key" to "value" for the specified * column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance - * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param writeOpts {@link org.forstdb.WriteOptions} instance. * @param key the specified key to be inserted. * @param value the value associated with the specified key. *

    @@ -1010,9 +1025,9 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, * Set the database entry for "key" to "value" for the specified * column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance - * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param writeOpts {@link org.forstdb.WriteOptions} instance. * @param key the specified key to be inserted. Position and limit is used. * Supports direct buffer only. * @param value the value associated with the specified key. Position and limit is used. @@ -1026,9 +1041,18 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, */ public void put(final ColumnFamilyHandle columnFamilyHandle, final WriteOptions writeOpts, final ByteBuffer key, final ByteBuffer value) throws RocksDBException { - assert key.isDirect() && value.isDirect(); - putDirect(nativeHandle_, writeOpts.nativeHandle_, key, key.position(), key.remaining(), value, - value.position(), value.remaining(), columnFamilyHandle.nativeHandle_); + if (key.isDirect() && value.isDirect()) { + putDirect(nativeHandle_, writeOpts.nativeHandle_, key, key.position(), key.remaining(), value, + value.position(), value.remaining(), columnFamilyHandle.nativeHandle_); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + put(nativeHandle_, writeOpts.nativeHandle_, key.array(), key.arrayOffset() + key.position(), + key.remaining(), value.array(), value.arrayOffset() + value.position(), value.remaining(), + columnFamilyHandle.nativeHandle_); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } key.position(key.limit()); value.position(value.limit()); } @@ -1036,7 +1060,7 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, final WriteOptions /** * Set the database entry for "key" to "value". * - * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param writeOpts {@link org.forstdb.WriteOptions} instance. * @param key the specified key to be inserted. Position and limit is used. * Supports direct buffer only. * @param value the value associated with the specified key. Position and limit is used. @@ -1050,9 +1074,18 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, final WriteOptions */ public void put(final WriteOptions writeOpts, final ByteBuffer key, final ByteBuffer value) throws RocksDBException { - assert key.isDirect() && value.isDirect(); - putDirect(nativeHandle_, writeOpts.nativeHandle_, key, key.position(), key.remaining(), value, - value.position(), value.remaining(), 0); + if (key.isDirect() && value.isDirect()) { + putDirect(nativeHandle_, writeOpts.nativeHandle_, key, key.position(), key.remaining(), value, + value.position(), value.remaining(), 0); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + put(nativeHandle_, writeOpts.nativeHandle_, key.array(), key.arrayOffset() + key.position(), + key.remaining(), value.array(), value.arrayOffset() + value.position(), + value.remaining()); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } key.position(key.limit()); value.position(value.limit()); } @@ -1061,9 +1094,9 @@ public void put(final WriteOptions writeOpts, final ByteBuffer key, final ByteBu * Set the database entry for "key" to "value" for the specified * column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance - * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param writeOpts {@link org.forstdb.WriteOptions} instance. * @param key The specified key to be inserted * @param offset the offset of the "key" array to be used, must be * non-negative and no larger than "key".length @@ -1084,8 +1117,8 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); put(nativeHandle_, writeOpts.nativeHandle_, key, offset, len, value, vOffset, vLen, columnFamilyHandle.nativeHandle_); } @@ -1128,7 +1161,7 @@ public void delete(final byte[] key, final int offset, final int len) * success, and a non-OK status on error. It is not an error if "key" * did not exist in the database. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key Key to delete within database * @@ -1145,7 +1178,7 @@ public void delete(final ColumnFamilyHandle columnFamilyHandle, * success, and a non-OK status on error. It is not an error if "key" * did not exist in the database. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key Key to delete within database * @param offset the offset of the "key" array to be used, @@ -1203,7 +1236,7 @@ public void delete(final WriteOptions writeOpt, final byte[] key, * success, and a non-OK status on error. It is not an error if "key" * did not exist in the database. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param writeOpt WriteOptions to be used with delete operation * @param key Key to delete within database @@ -1223,7 +1256,7 @@ public void delete(final ColumnFamilyHandle columnFamilyHandle, * success, and a non-OK status on error. It is not an error if "key" * did not exist in the database. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param writeOpt WriteOptions to be used with delete operation * @param key Key to delete within database @@ -1245,7 +1278,7 @@ public void delete(final ColumnFamilyHandle columnFamilyHandle, /** * Get the value associated with the specified key within column family. * - * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param opt {@link org.forstdb.ReadOptions} instance. * @param key the key to retrieve the value. It is using position and limit. * Supports direct buffer only. * @param value the out-value to receive the retrieved value. @@ -1263,9 +1296,18 @@ public void delete(final ColumnFamilyHandle columnFamilyHandle, */ public int get(final ReadOptions opt, final ByteBuffer key, final ByteBuffer value) throws RocksDBException { - assert key.isDirect() && value.isDirect(); - final int result = getDirect(nativeHandle_, opt.nativeHandle_, key, key.position(), - key.remaining(), value, value.position(), value.remaining(), 0); + final int result; + if (key.isDirect() && value.isDirect()) { + result = getDirect(nativeHandle_, opt.nativeHandle_, key, key.position(), key.remaining(), + value, value.position(), value.remaining(), 0); + } else if (!key.isDirect() && !value.isDirect()) { + result = + get(nativeHandle_, opt.nativeHandle_, key.array(), key.arrayOffset() + key.position(), + key.remaining(), value.array(), value.arrayOffset() + value.position(), + value.remaining(), defaultColumnFamilyHandle_.nativeHandle_); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } if (result != NOT_FOUND) { value.limit(Math.min(value.limit(), value.position() + result)); } @@ -1276,9 +1318,9 @@ public int get(final ReadOptions opt, final ByteBuffer key, final ByteBuffer val /** * Get the value associated with the specified key within column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance - * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param opt {@link org.forstdb.ReadOptions} instance. * @param key the key to retrieve the value. It is using position and limit. * Supports direct buffer only. * @param value the out-value to receive the retrieved value. @@ -1328,7 +1370,7 @@ public int get(final ColumnFamilyHandle columnFamilyHandle, final ReadOptions op * @throws RocksDBException thrown if error happens in underlying * native library. */ - @Experimental("Performance optimization for a very specific workload") + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) public void singleDelete(final byte[] key) throws RocksDBException { singleDelete(nativeHandle_, key, key.length); } @@ -1355,9 +1397,9 @@ public void singleDelete(final byte[] key) throws RocksDBException { * @throws RocksDBException thrown if error happens in underlying * native library. */ - @Experimental("Performance optimization for a very specific workload") - public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, - final byte[] key) throws RocksDBException { + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, final byte[] key) + throws RocksDBException { singleDelete(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_); } @@ -1386,9 +1428,8 @@ public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, * @throws RocksDBException thrown if error happens in underlying * native library. */ - @Experimental("Performance optimization for a very specific workload") - public void singleDelete(final WriteOptions writeOpt, final byte[] key) - throws RocksDBException { + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) + public void singleDelete(final WriteOptions writeOpt, final byte[] key) throws RocksDBException { singleDelete(nativeHandle_, writeOpt.nativeHandle_, key, key.length); } @@ -1417,9 +1458,9 @@ public void singleDelete(final WriteOptions writeOpt, final byte[] key) * @throws RocksDBException thrown if error happens in underlying * native library. */ - @Experimental("Performance optimization for a very specific workload") - public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, - final WriteOptions writeOpt, final byte[] key) throws RocksDBException { + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, final WriteOptions writeOpt, + final byte[] key) throws RocksDBException { singleDelete(nativeHandle_, writeOpt.nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_); } @@ -1454,7 +1495,7 @@ public void deleteRange(final byte[] beginKey, final byte[] endKey) * non-OK status on error. It is not an error if "key" did not exist in the * database. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} instance * @param beginKey First key to delete within database (inclusive) * @param endKey Last key to delete within database (exclusive) * @@ -1498,7 +1539,7 @@ public void deleteRange(final WriteOptions writeOpt, final byte[] beginKey, * non-OK status on error. It is not an error if "key" did not exist in the * database. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} instance * @param writeOpt WriteOptions to be used with delete operation * @param beginKey First key to delete within database (included) * @param endKey Last key to delete within database (excluded) @@ -1552,8 +1593,8 @@ public void merge(final byte[] key, final byte[] value) */ public void merge(final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); merge(nativeHandle_, key, offset, len, value, vOffset, vLen); } @@ -1597,8 +1638,8 @@ public void merge(final ColumnFamilyHandle columnFamilyHandle, public void merge(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); merge(nativeHandle_, key, offset, len, value, vOffset, vLen, columnFamilyHandle.nativeHandle_); } @@ -1644,12 +1685,48 @@ public void merge(final WriteOptions writeOpts, final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); merge(nativeHandle_, writeOpts.nativeHandle_, key, offset, len, value, vOffset, vLen); } + public void merge(final WriteOptions writeOpts, final ByteBuffer key, final ByteBuffer value) + throws RocksDBException { + if (key.isDirect() && value.isDirect()) { + mergeDirect(nativeHandle_, writeOpts.nativeHandle_, key, key.position(), key.remaining(), + value, value.position(), value.remaining(), 0); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + merge(nativeHandle_, writeOpts.nativeHandle_, key.array(), key.arrayOffset() + key.position(), + key.remaining(), value.array(), value.arrayOffset() + value.position(), + value.remaining()); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + key.position(key.limit()); + value.position(value.limit()); + } + + public void merge(final ColumnFamilyHandle columnFamilyHandle, final WriteOptions writeOpts, + final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + if (key.isDirect() && value.isDirect()) { + mergeDirect(nativeHandle_, writeOpts.nativeHandle_, key, key.position(), key.remaining(), + value, value.position(), value.remaining(), columnFamilyHandle.nativeHandle_); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + merge(nativeHandle_, writeOpts.nativeHandle_, key.array(), key.arrayOffset() + key.position(), + key.remaining(), value.array(), value.arrayOffset() + value.position(), value.remaining(), + columnFamilyHandle.nativeHandle_); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + key.position(key.limit()); + value.position(value.limit()); + } + /** * Delete the database entry (if any) for "key". Returns OK on * success, and a non-OK status on error. It is not an error if "key" @@ -1673,7 +1750,7 @@ public void delete(final WriteOptions writeOpt, final ByteBuffer key) throws Roc * success, and a non-OK status on error. It is not an error if "key" * did not exist in the database. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param writeOpt WriteOptions to be used with delete operation * @param key Key to delete within database. It is using position and limit. @@ -1736,8 +1813,8 @@ public void merge( final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); merge(nativeHandle_, writeOpts.nativeHandle_, key, offset, len, value, vOffset, vLen, columnFamilyHandle.nativeHandle_); @@ -1821,15 +1898,15 @@ public int get(final byte[] key, final byte[] value) throws RocksDBException { public int get(final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); return get(nativeHandle_, key, offset, len, value, vOffset, vLen); } /** * Get the value associated with the specified key within column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the key to retrieve the value. * @param value the out-value to receive the retrieved value. @@ -1852,7 +1929,7 @@ public int get(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, /** * Get the value associated with the specified key within column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the key to retrieve the value. * @param offset the offset of the "key" array to be used, must be @@ -1878,8 +1955,8 @@ public int get(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, public int get(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException, IllegalArgumentException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); return get(nativeHandle_, key, offset, len, value, vOffset, vLen, columnFamilyHandle.nativeHandle_); } @@ -1887,7 +1964,7 @@ public int get(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, /** * Get the value associated with the specified key. * - * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param opt {@link org.forstdb.ReadOptions} instance. * @param key the key to retrieve the value. * @param value the out-value to receive the retrieved value. * @return The size of the actual value that matches the specified @@ -1909,7 +1986,7 @@ public int get(final ReadOptions opt, final byte[] key, /** * Get the value associated with the specified key. * - * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param opt {@link org.forstdb.ReadOptions} instance. * @param key the key to retrieve the value. * @param offset the offset of the "key" array to be used, must be * non-negative and no larger than "key".length @@ -1933,8 +2010,8 @@ public int get(final ReadOptions opt, final byte[] key, public int get(final ReadOptions opt, final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); return get(nativeHandle_, opt.nativeHandle_, key, offset, len, value, vOffset, vLen); } @@ -1942,9 +2019,9 @@ public int get(final ReadOptions opt, final byte[] key, final int offset, /** * Get the value associated with the specified key within column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance - * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param opt {@link org.forstdb.ReadOptions} instance. * @param key the key to retrieve the value. * @param value the out-value to receive the retrieved value. * @return The size of the actual value that matches the specified @@ -1967,9 +2044,9 @@ public int get(final ColumnFamilyHandle columnFamilyHandle, /** * Get the value associated with the specified key within column family. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance - * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param opt {@link org.forstdb.ReadOptions} instance. * @param key the key to retrieve the value. * @param offset the offset of the "key" array to be used, must be * non-negative and no larger than "key".length @@ -1994,8 +2071,8 @@ public int get(final ColumnFamilyHandle columnFamilyHandle, final ReadOptions opt, final byte[] key, final int offset, final int len, final byte[] value, final int vOffset, final int vLen) throws RocksDBException { - checkBounds(offset, len, key.length); - checkBounds(vOffset, vLen, value.length); + CheckBounds(offset, len, key.length); + CheckBounds(vOffset, vLen, value.length); return get(nativeHandle_, opt.nativeHandle_, key, offset, len, value, vOffset, vLen, columnFamilyHandle.nativeHandle_); } @@ -2034,7 +2111,7 @@ public byte[] get(final byte[] key) throws RocksDBException { */ public byte[] get(final byte[] key, final int offset, final int len) throws RocksDBException { - checkBounds(offset, len, key.length); + CheckBounds(offset, len, key.length); return get(nativeHandle_, key, offset, len); } @@ -2043,7 +2120,7 @@ public byte[] get(final byte[] key, final int offset, * the value associated with the specified input key if any. null will be * returned if the specified key is not found. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the key retrieve the value. * @return a byte array storing the value associated with the input key if @@ -2063,7 +2140,7 @@ public byte[] get(final ColumnFamilyHandle columnFamilyHandle, * the value associated with the specified input key if any. null will be * returned if the specified key is not found. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the key retrieve the value. * @param offset the offset of the "key" array to be used, must be @@ -2079,7 +2156,7 @@ public byte[] get(final ColumnFamilyHandle columnFamilyHandle, public byte[] get(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final int offset, final int len) throws RocksDBException { - checkBounds(offset, len, key.length); + CheckBounds(offset, len, key.length); return get(nativeHandle_, key, offset, len, columnFamilyHandle.nativeHandle_); } @@ -2121,7 +2198,7 @@ public byte[] get(final ReadOptions opt, final byte[] key) */ public byte[] get(final ReadOptions opt, final byte[] key, final int offset, final int len) throws RocksDBException { - checkBounds(offset, len, key.length); + CheckBounds(offset, len, key.length); return get(nativeHandle_, opt.nativeHandle_, key, offset, len); } @@ -2130,7 +2207,7 @@ public byte[] get(final ReadOptions opt, final byte[] key, final int offset, * the value associated with the specified input key if any. null will be * returned if the specified key is not found. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the key retrieve the value. * @param opt Read options. @@ -2151,7 +2228,7 @@ public byte[] get(final ColumnFamilyHandle columnFamilyHandle, * the value associated with the specified input key if any. null will be * returned if the specified key is not found. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the key retrieve the value. * @param offset the offset of the "key" array to be used, must be @@ -2168,7 +2245,7 @@ public byte[] get(final ColumnFamilyHandle columnFamilyHandle, public byte[] get(final ColumnFamilyHandle columnFamilyHandle, final ReadOptions opt, final byte[] key, final int offset, final int len) throws RocksDBException { - checkBounds(offset, len, key.length); + CheckBounds(offset, len, key.length); return get(nativeHandle_, opt.nativeHandle_, key, offset, len, columnFamilyHandle.nativeHandle_); } @@ -2186,7 +2263,7 @@ public byte[] get(final ColumnFamilyHandle columnFamilyHandle, */ public List multiGetAsList(final List keys) throws RocksDBException { - assert(keys.size() != 0); + assert (!keys.isEmpty()); final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); final int[] keyOffsets = new int[keysArray.length]; @@ -2208,7 +2285,7 @@ public List multiGetAsList(final List keys) *

    * * @param columnFamilyHandleList {@link java.util.List} containing - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param keys List of keys for which values need to be retrieved. * @return List of values for the given list of keys. List will contain * null for keys which could not be found. @@ -2222,7 +2299,7 @@ public List multiGetAsList( final List columnFamilyHandleList, final List keys) throws RocksDBException, IllegalArgumentException { - assert(keys.size() != 0); + assert (!keys.isEmpty()); // Check if key size equals cfList size. If not a exception must be // thrown. If not a Segmentation fault happens. if (keys.size() != columnFamilyHandleList.size()) { @@ -2259,7 +2336,7 @@ public List multiGetAsList( */ public List multiGetAsList(final ReadOptions opt, final List keys) throws RocksDBException { - assert(keys.size() != 0); + assert (!keys.isEmpty()); final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); final int[] keyOffsets = new int[keysArray.length]; @@ -2282,7 +2359,7 @@ public List multiGetAsList(final ReadOptions opt, * * @param opt Read options. * @param columnFamilyHandleList {@link java.util.List} containing - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param keys of keys for which values need to be retrieved. * @return List of values for the given list of keys. List will contain * null for keys which could not be found. @@ -2295,7 +2372,7 @@ public List multiGetAsList(final ReadOptions opt, public List multiGetAsList(final ReadOptions opt, final List columnFamilyHandleList, final List keys) throws RocksDBException { - assert(keys.size() != 0); + assert (!keys.isEmpty()); // Check if key size equals cfList size. If not a exception must be // thrown. If not a Segmentation fault happens. if (keys.size()!=columnFamilyHandleList.size()){ @@ -2332,10 +2409,11 @@ public List multiGetAsList(final ReadOptions opt, */ public List multiGetByteBuffers( final List keys, final List values) throws RocksDBException { - final ReadOptions readOptions = new ReadOptions(); - final List columnFamilyHandleList = new ArrayList<>(1); - columnFamilyHandleList.add(getDefaultColumnFamily()); - return multiGetByteBuffers(readOptions, columnFamilyHandleList, keys, values); + try (ReadOptions readOptions = new ReadOptions()) { + final List columnFamilyHandleList = new ArrayList<>(1); + columnFamilyHandleList.add(getDefaultColumnFamily()); + return multiGetByteBuffers(readOptions, columnFamilyHandleList, keys, values); + } } /** @@ -2364,7 +2442,7 @@ public List multiGetByteBuffers(final ReadOptions readOptio *

    * * @param columnFamilyHandleList {@link java.util.List} containing - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param keys list of keys for which values need to be retrieved. * @param values list of buffers to return retrieved values in * @throws RocksDBException if error happens in underlying native library. @@ -2375,8 +2453,9 @@ public List multiGetByteBuffers(final ReadOptions readOptio public List multiGetByteBuffers( final List columnFamilyHandleList, final List keys, final List values) throws RocksDBException { - final ReadOptions readOptions = new ReadOptions(); - return multiGetByteBuffers(readOptions, columnFamilyHandleList, keys, values); + try (ReadOptions readOptions = new ReadOptions()) { + return multiGetByteBuffers(readOptions, columnFamilyHandleList, keys, values); + } } /** @@ -2388,7 +2467,7 @@ public List multiGetByteBuffers( * * @param readOptions Read options * @param columnFamilyHandleList {@link java.util.List} containing - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param keys list of keys for which values need to be retrieved. * @param values list of buffers to return retrieved values in * @throws RocksDBException if error happens in underlying native library. @@ -2399,7 +2478,7 @@ public List multiGetByteBuffers( public List multiGetByteBuffers(final ReadOptions readOptions, final List columnFamilyHandleList, final List keys, final List values) throws RocksDBException { - assert (keys.size() != 0); + assert (!keys.isEmpty()); // Check if key size equals cfList size. If not a exception must be // thrown. If not a Segmentation fault happens. @@ -2469,6 +2548,259 @@ public List multiGetByteBuffers(final ReadOptions readOptio return results; } + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * @param key byte array of a key to search for* + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final byte[] key) { + return keyExists(key, 0, key.length); + } + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * @param key byte array of a key to search for + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than "key".length + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final byte[] key, final int offset, final int len) { + return keyExists(null, null, key, offset, len); + } + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key byte array of a key to search for + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ColumnFamilyHandle columnFamilyHandle, final byte[] key) { + return keyExists(columnFamilyHandle, key, 0, key.length); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key byte array of a key to search for + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than "key".length + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final int offset, final int len) { + return keyExists(columnFamilyHandle, null, key, offset, len); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param readOptions {@link ReadOptions} instance + * @param key byte array of a key to search for + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ReadOptions readOptions, final byte[] key) { + return keyExists(readOptions, key, 0, key.length); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param readOptions {@link ReadOptions} instance + * @param key byte array of a key to search for + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than "key".length + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists( + final ReadOptions readOptions, final byte[] key, final int offset, final int len) { + return keyExists(null, readOptions, key, offset, len); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param readOptions {@link ReadOptions} instance + * @param key byte array of a key to search for + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions, final byte[] key) { + return keyExists(columnFamilyHandle, readOptions, key, 0, key.length); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param readOptions {@link ReadOptions} instance + * @param key byte array of a key to search for + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than "key".length + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions, final byte[] key, final int offset, final int len) { + checkBounds(offset, len, key.length); + return keyExists(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + readOptions == null ? 0 : readOptions.nativeHandle_, key, offset, len); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param key ByteBuffer with key. Must be allocated as direct. + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ByteBuffer key) { + return keyExists(null, null, key); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key ByteBuffer with key. Must be allocated as direct. + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key) { + return keyExists(columnFamilyHandle, null, key); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param readOptions {@link ReadOptions} instance + * @param key ByteBuffer with key. Must be allocated as direct. + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ReadOptions readOptions, final ByteBuffer key) { + return keyExists(null, readOptions, key); + } + + /** + * Check if a key exists in the database. + * This method is not as lightweight as {@code keyMayExist} but it gives a 100% guarantee + * of a correct result, whether the key exists or not. + * + * Internally it checks if the key may exist and then double checks with read operation + * that confirms the key exists. This deals with the case where {@code keyMayExist} may return + * a false positive. + * + * The code crosses the Java/JNI boundary only once. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param readOptions {@link ReadOptions} instance + * @param key ByteBuffer with key. Must be allocated as direct. + * @return true if key exist in database, otherwise false. + */ + public boolean keyExists(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions, final ByteBuffer key) { + assert key != null : "key ByteBuffer parameter cannot be null"; + assert key.isDirect() : "key parameter must be a direct ByteBuffer"; + + return keyExistsDirect(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + readOptions == null ? 0 : readOptions.nativeHandle_, key, key.position(), key.limit()); + } + /** * If the key definitely does not exist in the database, then this method * returns false, otherwise it returns true if the key might exist. @@ -2714,7 +3046,7 @@ public boolean keyMayExist( final ReadOptions readOptions, final byte[] key, final int offset, final int len, /* @Nullable */ final Holder valueHolder) { - checkBounds(offset, len, key.length); + CheckBounds(offset, len, key.length); if (valueHolder == null) { return keyMayExist(nativeHandle_, columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, @@ -2854,9 +3186,11 @@ public boolean keyMayExist(final ColumnFamilyHandle columnFamilyHandle, final ReadOptions readOptions, final ByteBuffer key) { assert key != null : "key ByteBuffer parameter cannot be null"; assert key.isDirect() : "key parameter must be a direct ByteBuffer"; - return keyMayExistDirect(nativeHandle_, + final boolean result = keyMayExistDirect(nativeHandle_, columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, readOptions == null ? 0 : readOptions.nativeHandle_, key, key.position(), key.limit()); + key.position(key.limit()); + return result; } /** @@ -2889,6 +3223,7 @@ public KeyMayExist keyMayExist(final ColumnFamilyHandle columnFamilyHandle, value, value.position(), value.remaining()); final int valueLength = result[1]; value.limit(value.position() + Math.min(valueLength, value.remaining())); + key.position(key.limit()); return new KeyMayExist(KeyMayExist.KeyMayExistEnum.values()[result[0]], valueLength); } @@ -2905,7 +3240,9 @@ public KeyMayExist keyMayExist(final ColumnFamilyHandle columnFamilyHandle, * @return instance of iterator object. */ public RocksIterator newIterator() { - return new RocksIterator(this, iterator(nativeHandle_)); + return new RocksIterator(this, + iterator(nativeHandle_, defaultColumnFamilyHandle_.nativeHandle_, + defaultReadOptions_.nativeHandle_)); } /** @@ -2922,8 +3259,9 @@ public RocksIterator newIterator() { * @return instance of iterator object. */ public RocksIterator newIterator(final ReadOptions readOptions) { - return new RocksIterator(this, iterator(nativeHandle_, - readOptions.nativeHandle_)); + return new RocksIterator(this, + iterator( + nativeHandle_, defaultColumnFamilyHandle_.nativeHandle_, readOptions.nativeHandle_)); } /** @@ -2936,14 +3274,15 @@ public RocksIterator newIterator(final ReadOptions readOptions) { * The returned iterator should be closed before this db is closed. *

    * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @return instance of iterator object. */ public RocksIterator newIterator( final ColumnFamilyHandle columnFamilyHandle) { - return new RocksIterator(this, iteratorCF(nativeHandle_, - columnFamilyHandle.nativeHandle_)); + return new RocksIterator(this, + iterator( + nativeHandle_, columnFamilyHandle.nativeHandle_, defaultReadOptions_.nativeHandle_)); } /** @@ -2956,15 +3295,15 @@ public RocksIterator newIterator( * The returned iterator should be closed before this db is closed. *

    * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param readOptions {@link ReadOptions} instance. * @return instance of iterator object. */ public RocksIterator newIterator(final ColumnFamilyHandle columnFamilyHandle, final ReadOptions readOptions) { - return new RocksIterator(this, iteratorCF(nativeHandle_, - columnFamilyHandle.nativeHandle_, readOptions.nativeHandle_)); + return new RocksIterator( + this, iterator(nativeHandle_, columnFamilyHandle.nativeHandle_, readOptions.nativeHandle_)); } /** @@ -2973,8 +3312,8 @@ public RocksIterator newIterator(final ColumnFamilyHandle columnFamilyHandle, * before the db is deleted * * @param columnFamilyHandleList {@link java.util.List} containing - * {@link org.rocksdb.ColumnFamilyHandle} instances. - * @return {@link java.util.List} containing {@link org.rocksdb.RocksIterator} + * {@link org.forstdb.ColumnFamilyHandle} instances. + * @return {@link java.util.List} containing {@link org.forstdb.RocksIterator} * instances * * @throws RocksDBException thrown if error happens in underlying @@ -2992,9 +3331,9 @@ public List newIterators( * before the db is deleted * * @param columnFamilyHandleList {@link java.util.List} containing - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param readOptions {@link ReadOptions} instance. - * @return {@link java.util.List} containing {@link org.rocksdb.RocksIterator} + * @return {@link java.util.List} containing {@link org.forstdb.RocksIterator} * instances * * @throws RocksDBException thrown if error happens in underlying @@ -3072,7 +3411,7 @@ public void releaseSnapshot(final Snapshot snapshot) { * of the sstables that make up the db contents. * * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family. * @param property to be fetched. See above for examples * @return property value @@ -3133,7 +3472,7 @@ public Map getMapProperty(final String property) /** * Gets a property map. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family. * @param property to be fetched. * @@ -3191,7 +3530,7 @@ public long getLongProperty(final String property) throws RocksDBException { *

    Java 8: In Java 8 the value should be treated as * unsigned long using provided methods of type {@link Long}.

    * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family * @param property to be fetched. * @@ -3257,7 +3596,7 @@ public long getAggregatedLongProperty(final String property) * should include the recently written data in the mem-tables (if * the mem-table type supports it), data serialized to disk, or both. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family * @param ranges the ranges over which to approximate sizes * @param sizeApproximationFlags flags to determine what to include in the @@ -3319,7 +3658,7 @@ public CountAndSize(final long count, final long size) { * {@link #getApproximateSizes(ColumnFamilyHandle, List, SizeApproximationFlag...)}, * except that it returns approximate number of records and size in memtables. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family * @param range the ranges over which to get the memtable stats * @@ -3380,7 +3719,7 @@ public void compactRange() throws RocksDBException { * * * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family. * * @throws RocksDBException thrown if an error occurs within the native @@ -3426,7 +3765,7 @@ public void compactRange(final byte[] begin, final byte[] end) *
  • {@link #compactRange(ColumnFamilyHandle)}
    • * * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family. * @param begin start of key range (included in range) * @param end end of key range (excluded from range) @@ -3449,7 +3788,7 @@ public void compactRange( * all data will have been pushed down to the last level containing * any data.

    * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance. + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} instance. * @param begin start of key range (included in range) * @param end end of key range (excluded from range) * @param compactRangeOptions options for the compaction @@ -3468,10 +3807,30 @@ public void compactRange( columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); } + /** + * ClipColumnFamily() will clip the entries in the CF according to the range + * [begin_key, end_key). Returns OK on success, and a non-OK status on error. + * Any entries outside this range will be completely deleted (including + * tombstones). + * + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} instance + * @param beginKey First key to clip within database (inclusive) + * @param endKey Last key to clip within database (exclusive) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void clipColumnFamily(final ColumnFamilyHandle columnFamilyHandle, final byte[] beginKey, + final byte[] endKey) throws RocksDBException { + clipColumnFamily(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, beginKey, 0, + beginKey.length, endKey, 0, endKey.length); + } + /** * Change the options for the column family handle. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family. * @param mutableColumnFamilyOptions the options. * @@ -3485,10 +3844,44 @@ public void setOptions( mutableColumnFamilyOptions.getKeys(), mutableColumnFamilyOptions.getValues()); } + /** + * Set performance level for rocksdb performance measurement. + * @param level + * @throws IllegalArgumentException for UNINITIALIZED and OUT_OF_BOUNDS values + * as they can't be used for settings. + */ + public void setPerfLevel(final PerfLevel level) { + if (level == PerfLevel.UNINITIALIZED) { + throw new IllegalArgumentException("Unable to set UNINITIALIZED level"); + } else if (level == PerfLevel.OUT_OF_BOUNDS) { + throw new IllegalArgumentException("Unable to set OUT_OF_BOUNDS level"); + } else { + setPerfLevel(level.getValue()); + } + } + + /** + * Return current performance level measurement settings. + * @return + */ + public PerfLevel getPerfLevel() { + byte level = getPerfLevelNative(); + return PerfLevel.getPerfLevel(level); + } + + /** + * Return perf context bound to this thread. + * @return + */ + public PerfContext getPerfContext() { + long native_handle = getPerfContextNative(); + return new PerfContext(native_handle); + } + /** * Get the options for the column family handle * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance, or null for the default column family. * * @return the options parsed from the options string return by RocksDB @@ -3784,13 +4177,13 @@ public Env getEnv() { * is not GC'ed before this method finishes. If the wait parameter is * set to false, flush processing is asynchronous.

    * - * @param flushOptions {@link org.rocksdb.FlushOptions} instance. + * @param flushOptions {@link org.forstdb.FlushOptions} instance. * @throws RocksDBException thrown if an error occurs within the native * part of the library. */ public void flush(final FlushOptions flushOptions) throws RocksDBException { - flush(flushOptions, (List) null); + flush(flushOptions, Collections.singletonList(getDefaultColumnFamily())); } /** @@ -3800,8 +4193,8 @@ public void flush(final FlushOptions flushOptions) * is not GC'ed before this method finishes. If the wait parameter is * set to false, flush processing is asynchronous.

    * - * @param flushOptions {@link org.rocksdb.FlushOptions} instance. - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance. + * @param flushOptions {@link org.forstdb.FlushOptions} instance. + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} instance. * @throws RocksDBException thrown if an error occurs within the native * part of the library. */ @@ -3822,7 +4215,7 @@ public void flush(final FlushOptions flushOptions, * specified up to the latest sequence number at the time when flush is * requested. * - * @param flushOptions {@link org.rocksdb.FlushOptions} instance. + * @param flushOptions {@link org.forstdb.FlushOptions} instance. * @param columnFamilyHandles column family handles. * @throws RocksDBException thrown if an error occurs within the native * part of the library. @@ -3886,7 +4279,7 @@ public void disableFileDeletions() throws RocksDBException { } /** - *

    Allow compactions to delete obsolete files. + *

    Enable deleting obsolete files. * If force == true, the call to EnableFileDeletions() * will guarantee that file deletions are enabled after * the call, even if DisableFileDeletions() was called @@ -4003,9 +4396,9 @@ public List getSortedWalFiles() throws RocksDBException { * * @param sequenceNumber sequence number offset * - * @return {@link org.rocksdb.TransactionLogIterator} instance. + * @return {@link org.forstdb.TransactionLogIterator} instance. * - * @throws org.rocksdb.RocksDBException if iterator cannot be retrieved + * @throws org.forstdb.RocksDBException if iterator cannot be retrieved * from native-side. */ public TransactionLogIterator getUpdatesSince(final long sequenceNumber) @@ -4125,8 +4518,17 @@ public void verifyChecksum() throws RocksDBException { * @return The handle of the default column family */ public ColumnFamilyHandle getDefaultColumnFamily() { - final ColumnFamilyHandle cfHandle = new ColumnFamilyHandle(this, - getDefaultColumnFamily(nativeHandle_)); + return defaultColumnFamilyHandle_; + } + + /** + * Create a handle for the default CF on open + * + * @return the default family handle + */ + protected ColumnFamilyHandle makeDefaultColumnFamilyHandle() { + final ColumnFamilyHandle cfHandle = + new ColumnFamilyHandle(this, getDefaultColumnFamily(nativeHandle_)); cfHandle.disOwnNativeHandle(); return cfHandle; } @@ -4316,10 +4718,9 @@ public void tryCatchUpWithPrimary() throws RocksDBException { * @throws RocksDBException thrown if error happens in underlying * native library. */ - public void deleteFilesInRanges(final ColumnFamilyHandle columnFamily, - final List ranges, final boolean includeEnd) - throws RocksDBException { - if (ranges.size() == 0) { + public void deleteFilesInRanges(final ColumnFamilyHandle columnFamily, final List ranges, + final boolean includeEnd) throws RocksDBException { + if (ranges.isEmpty()) { return; } if ((ranges.size() % 2) != 0) { @@ -4338,7 +4739,7 @@ public void deleteFilesInRanges(final ColumnFamilyHandle columnFamily, * Be very careful using this method. * * @param path the path to the Rocksdb database. - * @param options {@link org.rocksdb.Options} instance. + * @param options {@link org.forstdb.Options} instance. * * @throws RocksDBException thrown if error happens in underlying * native library. @@ -4351,7 +4752,7 @@ public static void destroyDB(final String path, final Options options) private /* @Nullable */ long[] toNativeHandleList( /* @Nullable */ final List objectList) { if (objectList == null) { - return null; + return new long[0]; } final int len = objectList.size(); final long[] handleList = new long[len]; @@ -4361,6 +4762,7 @@ public static void destroyDB(final String path, final Options options) return handleList; } + @SuppressWarnings({"PMD.ForLoopVariableCount", "PMD.AvoidReassigningLoopVariables"}) private static long[] toRangeSliceHandles(final List ranges) { final long[] rangeSliceHandles = new long[ranges.size() * 2]; for (int i = 0, j = 0; i < ranges.size(); i++) { @@ -4375,18 +4777,16 @@ protected void storeOptionsInstance(final DBOptionsInterface options) { options_ = options; } - private static void checkBounds(final int offset, final int len, final int size) { + protected void storeDefaultColumnFamilyHandle(ColumnFamilyHandle columnFamilyHandle) { + defaultColumnFamilyHandle_ = columnFamilyHandle; + } + + private static void checkBounds(int offset, int len, int size) { if ((offset | len | (offset + len) | (size - (offset + len))) < 0) { throw new IndexOutOfBoundsException(String.format("offset(%d), len(%d), size(%d)", offset, len, size)); } } - private static int computeCapacityHint(final int estimatedNumberOfItems) { - // Default load factor for HashMap is 0.75, so N * 1.5 will be at the load - // limit. We add +1 for a buffer. - return (int)Math.ceil(estimatedNumberOfItems * 1.5 + 1.0); - } - // native methods private static native long open(final long optionsHandle, final String path) throws RocksDBException; @@ -4448,7 +4848,8 @@ private native long[] createColumnFamilies( throws RocksDBException; private native long createColumnFamilyWithImport(final long handle, final byte[] columnFamilyName, final int columnFamilyNamelen, final long columnFamilyOptions, - final long importColumnFamilyOptions, final long[] metadataHandeList) throws RocksDBException; + final long importColumnFamilyOptions, final long[] metadataHandleList) + throws RocksDBException; private native void dropColumnFamily( final long handle, final long cfHandle) throws RocksDBException; private native void dropColumnFamilies(final long handle, @@ -4507,6 +4908,9 @@ private native void deleteRange( final int beginKeyOffset, final int beginKeyLength, final byte[] endKey, final int endKeyOffset, final int endKeyLength, final long cfHandle) throws RocksDBException; + private native void clipColumnFamily(final long handle, final long cfHandle, + final byte[] beginKey, final int beginKeyOffset, final int beginKeyLength, + final byte[] endKey, final int endKeyOffset, final int endKeyLength) throws RocksDBException; private native void merge(final long handle, final byte[] key, final int keyOffset, final int keyLength, final byte[] value, final int valueOffset, final int valueLength) throws RocksDBException; @@ -4522,6 +4926,10 @@ private native void merge(final long handle, final long writeOptHandle, final byte[] key, final int keyOffset, final int keyLength, final byte[] value, final int valueOffset, final int valueLength, final long cfHandle) throws RocksDBException; + private native void mergeDirect(long handle, long writeOptHandle, ByteBuffer key, int keyOffset, + int keyLength, ByteBuffer value, int valueOffset, int valueLength, long cfHandle) + throws RocksDBException; + private native void write0(final long handle, final long writeOptHandle, final long wbHandle) throws RocksDBException; private native void write1(final long handle, final long writeOptHandle, @@ -4568,6 +4976,12 @@ private native void multiGet(final long dbHandle, final long rOptHandle, final int[] keyLengths, final ByteBuffer[] valuesArray, final int[] valuesSizeArray, final Status[] statusArray); + private native boolean keyExists(final long handle, final long cfHandle, final long readOptHandle, + final byte[] key, final int keyOffset, final int keyLength); + + private native boolean keyExistsDirect(final long handle, final long cfHandle, + final long readOptHandle, final ByteBuffer key, final int keyOffset, final int keyLength); + private native boolean keyMayExist( final long handle, final long cfHandle, final long readOptHandle, final byte[] key, final int keyOffset, final int keyLength); @@ -4577,11 +4991,7 @@ private native byte[][] keyMayExistFoundValue( private native void putDirect(long handle, long writeOptHandle, ByteBuffer key, int keyOffset, int keyLength, ByteBuffer value, int valueOffset, int valueLength, long cfHandle) throws RocksDBException; - private native long iterator(final long handle); - private native long iterator(final long handle, final long readOptHandle); - private native long iteratorCF(final long handle, final long cfHandle); - private native long iteratorCF(final long handle, final long cfHandle, - final long readOptHandle); + private native long iterator(final long handle, final long cfHandle, final long readOptHandle); private native long[] iterators(final long handle, final long[] columnFamilyHandles, final long readOptHandle) throws RocksDBException; @@ -4628,6 +5038,11 @@ private native void setOptions(final long handle, final long cfHandle, private native void setDBOptions(final long handle, final String[] keys, final String[] values) throws RocksDBException; private native String getDBOptions(final long handle); + private native void setPerfLevel(final byte level); + private native byte getPerfLevelNative(); + + private native long getPerfContextNative(); + private native String[] compactFiles(final long handle, final long compactionOptionsHandle, final long columnFamilyHandle, @@ -4730,12 +5145,10 @@ public String toString() { return getMajor() + "." + getMinor() + "." + getPatch(); } - private static Version fromEncodedVersion(int encodedVersion) { + private static Version fromEncodedVersion(final int encodedVersion) { final byte patch = (byte) (encodedVersion & 0xff); - encodedVersion >>= 8; - final byte minor = (byte) (encodedVersion & 0xff); - encodedVersion >>= 8; - final byte major = (byte) (encodedVersion & 0xff); + final byte minor = (byte) (encodedVersion >> 8 & 0xff); + final byte major = (byte) (encodedVersion >> 16 & 0xff); return new Version(major, minor, patch); } diff --git a/java/src/main/java/org/rocksdb/RocksDBException.java b/java/src/main/java/org/forstdb/RocksDBException.java similarity index 93% rename from java/src/main/java/org/rocksdb/RocksDBException.java rename to java/src/main/java/org/forstdb/RocksDBException.java index 8b035f458..c1f698d5a 100644 --- a/java/src/main/java/org/rocksdb/RocksDBException.java +++ b/java/src/main/java/org/forstdb/RocksDBException.java @@ -3,14 +3,14 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * A RocksDBException encapsulates the error of an operation. This exception * type is used to describe an internal error from the c++ rocksdb library. */ public class RocksDBException extends Exception { - + private static final long serialVersionUID = -5187634878466267120L; /* @Nullable */ private final Status status; /** diff --git a/java/src/main/java/org/rocksdb/RocksEnv.java b/java/src/main/java/org/forstdb/RocksEnv.java similarity index 98% rename from java/src/main/java/org/rocksdb/RocksEnv.java rename to java/src/main/java/org/forstdb/RocksEnv.java index ca010c9f9..5d6d1a639 100644 --- a/java/src/main/java/org/rocksdb/RocksEnv.java +++ b/java/src/main/java/org/forstdb/RocksEnv.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** *

    A RocksEnv is an interface used by the rocksdb implementation to access diff --git a/java/src/main/java/org/rocksdb/RocksIterator.java b/java/src/main/java/org/forstdb/RocksIterator.java similarity index 62% rename from java/src/main/java/org/rocksdb/RocksIterator.java rename to java/src/main/java/org/forstdb/RocksIterator.java index 20e56d2eb..8127ff157 100644 --- a/java/src/main/java/org/rocksdb/RocksIterator.java +++ b/java/src/main/java/org/forstdb/RocksIterator.java @@ -3,7 +3,9 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; + +import static org.forstdb.util.BufferUtil.CheckBounds; import java.nio.ByteBuffer; @@ -18,7 +20,7 @@ * non-const method, all threads accessing the same RocksIterator must use * external synchronization.

    * - * @see org.rocksdb.RocksObject + * @see org.forstdb.RocksObject */ public class RocksIterator extends AbstractRocksIterator { protected RocksIterator(final RocksDB rocksDB, final long nativeHandle) { @@ -39,6 +41,45 @@ public byte[] key() { return key0(nativeHandle_); } + /** + *

    Return the key for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @param key the out-value to receive the retrieved key. + * @return The size of the actual key. If the return key is greater than + * the length of the buffer {@code key}, then it indicates that the size of the + * input buffer {@code key} is insufficient and partial result will + * be returned. + */ + public int key(final byte[] key) { + assert isOwningHandle(); + return keyByteArray0(nativeHandle_, key, 0, key.length); + } + + /** + *

    Return the key for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @param key the out-value to receive the retrieved key. + * @param offset in {@code key} at which to place the retrieved key + * @param len limit to length of received key returned + * @return The size of the actual key. If the return key is greater than + * {@code len}, then it indicates that the size of the + * input buffer {@code key} is insufficient and partial result will + * be returned. + */ + public int key(final byte[] key, final int offset, final int len) { + assert isOwningHandle(); + CheckBounds(offset, len, key.length); + return keyByteArray0(nativeHandle_, key, offset, len); + } + /** *

    Return the key for the current entry. The underlying storage for * the returned slice is valid only until the next modification of @@ -48,7 +89,6 @@ public byte[] key() { * * @param key the out-value to receive the retrieved key. * It is using position and limit. Limit is set according to key size. - * Supports direct buffer only. * @return The size of the actual key. If the return key is greater than the * length of {@code key}, then it indicates that the size of the * input buffer {@code key} is insufficient and partial result will @@ -90,7 +130,6 @@ public byte[] value() { * * @param value the out-value to receive the retrieved value. * It is using position and limit. Limit is set according to value size. - * Supports direct buffer only. * @return The size of the actual value. If the return value is greater than the * length of {@code value}, then it indicates that the size of the * input buffer {@code value} is insufficient and partial result will @@ -110,6 +149,45 @@ public int value(final ByteBuffer value) { return result; } + /** + *

    Return the value for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @param value the out-value to receive the retrieved value. + * @return The size of the actual value. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. + */ + public int value(final byte[] value) { + assert isOwningHandle(); + return valueByteArray0(nativeHandle_, value, 0, value.length); + } + + /** + *

    Return the value for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @param value the out-value to receive the retrieved value. + * @param offset the offset within value at which to place the result + * @param len the length available in value after offset, for placing the result + * @return The size of the actual value. If the return value is greater than {@code len}, + * then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. + */ + public int value(final byte[] value, final int offset, final int len) { + assert isOwningHandle(); + CheckBounds(offset, len, value.length); + return valueByteArray0(nativeHandle_, value, offset, len); + } + @Override protected final native void disposeInternal(final long handle); @Override final native boolean isValid0(long handle); @Override final native void seekToFirst0(long handle); diff --git a/java/src/main/java/org/rocksdb/RocksIteratorInterface.java b/java/src/main/java/org/forstdb/RocksIteratorInterface.java similarity index 98% rename from java/src/main/java/org/rocksdb/RocksIteratorInterface.java rename to java/src/main/java/org/forstdb/RocksIteratorInterface.java index 819c21c2c..9d344c22b 100644 --- a/java/src/main/java/org/rocksdb/RocksIteratorInterface.java +++ b/java/src/main/java/org/forstdb/RocksIteratorInterface.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; @@ -18,7 +18,7 @@ * non-const method, all threads accessing the same RocksIterator must use * external synchronization.

    * - * @see org.rocksdb.RocksObject + * @see org.forstdb.RocksObject */ public interface RocksIteratorInterface { diff --git a/java/src/main/java/org/rocksdb/RocksMemEnv.java b/java/src/main/java/org/forstdb/RocksMemEnv.java similarity index 97% rename from java/src/main/java/org/rocksdb/RocksMemEnv.java rename to java/src/main/java/org/forstdb/RocksMemEnv.java index 39a6f6e1c..05db05900 100644 --- a/java/src/main/java/org/rocksdb/RocksMemEnv.java +++ b/java/src/main/java/org/forstdb/RocksMemEnv.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Memory environment. diff --git a/java/src/main/java/org/rocksdb/RocksMutableObject.java b/java/src/main/java/org/forstdb/RocksMutableObject.java similarity index 99% rename from java/src/main/java/org/rocksdb/RocksMutableObject.java rename to java/src/main/java/org/forstdb/RocksMutableObject.java index eb3215290..7840cc14b 100644 --- a/java/src/main/java/org/rocksdb/RocksMutableObject.java +++ b/java/src/main/java/org/forstdb/RocksMutableObject.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * RocksMutableObject is an implementation of {@link AbstractNativeReference} diff --git a/java/src/main/java/org/rocksdb/RocksObject.java b/java/src/main/java/org/forstdb/RocksObject.java similarity index 98% rename from java/src/main/java/org/rocksdb/RocksObject.java rename to java/src/main/java/org/forstdb/RocksObject.java index f07e1018a..7abd061fa 100644 --- a/java/src/main/java/org/rocksdb/RocksObject.java +++ b/java/src/main/java/org/forstdb/RocksObject.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * RocksObject is an implementation of {@link AbstractNativeReference} which diff --git a/java/src/main/java/org/rocksdb/SanityLevel.java b/java/src/main/java/org/forstdb/SanityLevel.java similarity index 98% rename from java/src/main/java/org/rocksdb/SanityLevel.java rename to java/src/main/java/org/forstdb/SanityLevel.java index 30568c363..4487d7d59 100644 --- a/java/src/main/java/org/rocksdb/SanityLevel.java +++ b/java/src/main/java/org/forstdb/SanityLevel.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum SanityLevel { NONE((byte) 0x0), diff --git a/java/src/main/java/org/rocksdb/SizeApproximationFlag.java b/java/src/main/java/org/forstdb/SizeApproximationFlag.java similarity index 96% rename from java/src/main/java/org/rocksdb/SizeApproximationFlag.java rename to java/src/main/java/org/forstdb/SizeApproximationFlag.java index fe3c2dd05..c39824925 100644 --- a/java/src/main/java/org/rocksdb/SizeApproximationFlag.java +++ b/java/src/main/java/org/forstdb/SizeApproximationFlag.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import java.util.List; diff --git a/java/src/main/java/org/rocksdb/SkipListMemTableConfig.java b/java/src/main/java/org/forstdb/SkipListMemTableConfig.java similarity index 98% rename from java/src/main/java/org/rocksdb/SkipListMemTableConfig.java rename to java/src/main/java/org/forstdb/SkipListMemTableConfig.java index e2c1b97d8..5e6f7090c 100644 --- a/java/src/main/java/org/rocksdb/SkipListMemTableConfig.java +++ b/java/src/main/java/org/forstdb/SkipListMemTableConfig.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** * The config for skip-list memtable representation. diff --git a/java/src/main/java/org/rocksdb/Slice.java b/java/src/main/java/org/forstdb/Slice.java similarity index 96% rename from java/src/main/java/org/rocksdb/Slice.java rename to java/src/main/java/org/forstdb/Slice.java index 6a01374d6..386ab4740 100644 --- a/java/src/main/java/org/rocksdb/Slice.java +++ b/java/src/main/java/org/forstdb/Slice.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** *

    Base class for slices which will receive @@ -11,7 +11,7 @@ * *

    byte[] backed slices typically perform better with * small keys and values. When using larger keys and - * values consider using {@link org.rocksdb.DirectSlice}

    + * values consider using {@link org.forstdb.DirectSlice}

    */ public class Slice extends AbstractSlice { @@ -27,7 +27,7 @@ public class Slice extends AbstractSlice { * at creation time.

    * *

    Note: You should be aware that - * {@see org.rocksdb.RocksObject#disOwnNativeHandle()} is intentionally + * {@see org.forstdb.RocksObject#disOwnNativeHandle()} is intentionally * called from the default Slice constructor, and that it is marked as * private. This is so that developers cannot construct their own default * Slice objects (at present). As developers cannot construct their own diff --git a/java/src/main/java/org/rocksdb/Snapshot.java b/java/src/main/java/org/forstdb/Snapshot.java similarity index 98% rename from java/src/main/java/org/rocksdb/Snapshot.java rename to java/src/main/java/org/forstdb/Snapshot.java index 1f471bd31..af5bb1ef8 100644 --- a/java/src/main/java/org/rocksdb/Snapshot.java +++ b/java/src/main/java/org/forstdb/Snapshot.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Snapshot of database diff --git a/java/src/main/java/org/rocksdb/SstFileManager.java b/java/src/main/java/org/forstdb/SstFileManager.java similarity index 99% rename from java/src/main/java/org/rocksdb/SstFileManager.java rename to java/src/main/java/org/forstdb/SstFileManager.java index 0b9a60061..ad51d753f 100644 --- a/java/src/main/java/org/rocksdb/SstFileManager.java +++ b/java/src/main/java/org/forstdb/SstFileManager.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Map; diff --git a/java/src/main/java/org/rocksdb/SstFileMetaData.java b/java/src/main/java/org/forstdb/SstFileMetaData.java similarity index 79% rename from java/src/main/java/org/rocksdb/SstFileMetaData.java rename to java/src/main/java/org/forstdb/SstFileMetaData.java index a04d05cb5..4fa210a9a 100644 --- a/java/src/main/java/org/rocksdb/SstFileMetaData.java +++ b/java/src/main/java/org/forstdb/SstFileMetaData.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The metadata that describes a SST file. @@ -20,6 +20,7 @@ public class SstFileMetaData { private final boolean beingCompacted; private final long numEntries; private final long numDeletions; + private final byte[] fileChecksum; /** * Called from JNI C++ @@ -35,19 +36,13 @@ public class SstFileMetaData { * @param beingCompacted true if the file is being compacted, false otherwise * @param numEntries the number of entries * @param numDeletions the number of deletions + * @param fileChecksum the full file checksum (if enabled) */ - protected SstFileMetaData( - final String fileName, - final String path, - final long size, - final long smallestSeqno, - final long largestSeqno, - final byte[] smallestKey, - final byte[] largestKey, - final long numReadsSampled, - final boolean beingCompacted, - final long numEntries, - final long numDeletions) { + @SuppressWarnings("PMD.ArrayIsStoredDirectly") + protected SstFileMetaData(final String fileName, final String path, final long size, + final long smallestSeqno, final long largestSeqno, final byte[] smallestKey, + final byte[] largestKey, final long numReadsSampled, final boolean beingCompacted, + final long numEntries, final long numDeletions, final byte[] fileChecksum) { this.fileName = fileName; this.path = path; this.size = size; @@ -59,6 +54,7 @@ protected SstFileMetaData( this.beingCompacted = beingCompacted; this.numEntries = numEntries; this.numDeletions = numDeletions; + this.fileChecksum = fileChecksum; } /** @@ -111,6 +107,7 @@ public long largestSeqno() { * * @return the smallest user defined key */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public byte[] smallestKey() { return smallestKey; } @@ -120,6 +117,7 @@ public byte[] smallestKey() { * * @return the largest user defined key */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public byte[] largestKey() { return largestKey; } @@ -159,4 +157,14 @@ public long numEntries() { public long numDeletions() { return numDeletions; } + + /** + * Get the full file checksum iff full file checksum is enabled. + * + * @return the file's checksum + */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") + public byte[] fileChecksum() { + return fileChecksum; + } } diff --git a/java/src/main/java/org/rocksdb/SstFileReader.java b/java/src/main/java/org/forstdb/SstFileReader.java similarity index 97% rename from java/src/main/java/org/rocksdb/SstFileReader.java rename to java/src/main/java/org/forstdb/SstFileReader.java index 678c3519c..2134f3d24 100644 --- a/java/src/main/java/org/rocksdb/SstFileReader.java +++ b/java/src/main/java/org/forstdb/SstFileReader.java @@ -3,13 +3,9 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class SstFileReader extends RocksObject { - static { - RocksDB.loadLibrary(); - } - public SstFileReader(final Options options) { super(newSstFileReader(options.nativeHandle_)); } diff --git a/java/src/main/java/org/rocksdb/SstFileReaderIterator.java b/java/src/main/java/org/forstdb/SstFileReaderIterator.java similarity index 99% rename from java/src/main/java/org/rocksdb/SstFileReaderIterator.java rename to java/src/main/java/org/forstdb/SstFileReaderIterator.java index a4a08167b..c1eac960d 100644 --- a/java/src/main/java/org/rocksdb/SstFileReaderIterator.java +++ b/java/src/main/java/org/forstdb/SstFileReaderIterator.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; diff --git a/java/src/main/java/org/rocksdb/SstFileWriter.java b/java/src/main/java/org/forstdb/SstFileWriter.java similarity index 96% rename from java/src/main/java/org/rocksdb/SstFileWriter.java rename to java/src/main/java/org/forstdb/SstFileWriter.java index 5dd0b6dd5..8e87b55ed 100644 --- a/java/src/main/java/org/rocksdb/SstFileWriter.java +++ b/java/src/main/java/org/forstdb/SstFileWriter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; @@ -13,15 +13,11 @@ * sequence number = 0. */ public class SstFileWriter extends RocksObject { - static { - RocksDB.loadLibrary(); - } - /** * SstFileWriter Constructor. * - * @param envOptions {@link org.rocksdb.EnvOptions} instance. - * @param options {@link org.rocksdb.Options} instance. + * @param envOptions {@link org.forstdb.EnvOptions} instance. + * @param options {@link org.forstdb.Options} instance. */ public SstFileWriter(final EnvOptions envOptions, final Options options) { super(newSstFileWriter( @@ -199,6 +195,8 @@ public long fileSize() throws RocksDBException { return fileSize(nativeHandle_); } + @SuppressWarnings("PMD.UnusedPrivateMethod") + // (AP) Should we expose a constructor wrapping this ? private static native long newSstFileWriter(final long envOptionsHandle, final long optionsHandle, final long userComparatorHandle, final byte comparatorType); diff --git a/java/src/main/java/org/rocksdb/SstPartitionerFactory.java b/java/src/main/java/org/forstdb/SstPartitionerFactory.java similarity index 96% rename from java/src/main/java/org/rocksdb/SstPartitionerFactory.java rename to java/src/main/java/org/forstdb/SstPartitionerFactory.java index ea6f13565..9fa9e32a5 100644 --- a/java/src/main/java/org/rocksdb/SstPartitionerFactory.java +++ b/java/src/main/java/org/forstdb/SstPartitionerFactory.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Handle to factory for SstPartitioner. It is used in {@link ColumnFamilyOptions} diff --git a/java/src/main/java/org/rocksdb/SstPartitionerFixedPrefixFactory.java b/java/src/main/java/org/forstdb/SstPartitionerFixedPrefixFactory.java similarity index 97% rename from java/src/main/java/org/rocksdb/SstPartitionerFixedPrefixFactory.java rename to java/src/main/java/org/forstdb/SstPartitionerFixedPrefixFactory.java index b1ccf08c1..c86eda32b 100644 --- a/java/src/main/java/org/rocksdb/SstPartitionerFixedPrefixFactory.java +++ b/java/src/main/java/org/forstdb/SstPartitionerFixedPrefixFactory.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Fixed prefix factory. It partitions SST files using fixed prefix of the key. diff --git a/java/src/main/java/org/rocksdb/StateType.java b/java/src/main/java/org/forstdb/StateType.java similarity index 98% rename from java/src/main/java/org/rocksdb/StateType.java rename to java/src/main/java/org/forstdb/StateType.java index 803fa37d9..2e81a1b73 100644 --- a/java/src/main/java/org/rocksdb/StateType.java +++ b/java/src/main/java/org/forstdb/StateType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The type used to refer to a thread state. diff --git a/java/src/main/java/org/rocksdb/Statistics.java b/java/src/main/java/org/forstdb/Statistics.java similarity index 92% rename from java/src/main/java/org/rocksdb/Statistics.java rename to java/src/main/java/org/forstdb/Statistics.java index 9f3c9a62c..33bfd2e21 100644 --- a/java/src/main/java/org/rocksdb/Statistics.java +++ b/java/src/main/java/org/forstdb/Statistics.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.EnumSet; @@ -14,7 +14,7 @@ public class Statistics extends RocksObject { public Statistics() { - super(newStatistics()); + super(newStatisticsInstance()); } public Statistics(final Statistics otherStatistics) { @@ -22,7 +22,7 @@ public Statistics(final Statistics otherStatistics) { } public Statistics(final EnumSet ignoreHistograms) { - super(newStatistics(toArrayValues(ignoreHistograms))); + super(newStatisticsInstance(toArrayValues(ignoreHistograms))); } public Statistics(final EnumSet ignoreHistograms, final Statistics otherStatistics) { @@ -134,8 +134,16 @@ public String toString() { return toString(nativeHandle_); } + private static long newStatisticsInstance() { + RocksDB.loadLibrary(); + return newStatistics(); + } private static native long newStatistics(); private static native long newStatistics(final long otherStatisticsHandle); + private static long newStatisticsInstance(final byte[] ignoreHistograms) { + RocksDB.loadLibrary(); + return newStatistics(ignoreHistograms); + } private static native long newStatistics(final byte[] ignoreHistograms); private static native long newStatistics( final byte[] ignoreHistograms, final long otherStatisticsHandle); diff --git a/java/src/main/java/org/rocksdb/StatisticsCollector.java b/java/src/main/java/org/forstdb/StatisticsCollector.java similarity index 98% rename from java/src/main/java/org/rocksdb/StatisticsCollector.java rename to java/src/main/java/org/forstdb/StatisticsCollector.java index fd00f85b2..8b698188c 100644 --- a/java/src/main/java/org/rocksdb/StatisticsCollector.java +++ b/java/src/main/java/org/forstdb/StatisticsCollector.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.List; import java.util.concurrent.Executors; @@ -61,6 +61,7 @@ public void shutDown(final int shutdownTimeout) throws InterruptedException { _executorService.awaitTermination(shutdownTimeout, TimeUnit.MILLISECONDS); } + @SuppressWarnings("PMD.CloseResource") private Runnable collectStatistics() { return () -> { while (_isRunning) { diff --git a/java/src/main/java/org/rocksdb/StatisticsCollectorCallback.java b/java/src/main/java/org/forstdb/StatisticsCollectorCallback.java similarity index 98% rename from java/src/main/java/org/rocksdb/StatisticsCollectorCallback.java rename to java/src/main/java/org/forstdb/StatisticsCollectorCallback.java index bed7828e0..8504b06ac 100644 --- a/java/src/main/java/org/rocksdb/StatisticsCollectorCallback.java +++ b/java/src/main/java/org/forstdb/StatisticsCollectorCallback.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Callback interface provided to StatisticsCollector. diff --git a/java/src/main/java/org/rocksdb/StatsCollectorInput.java b/java/src/main/java/org/forstdb/StatsCollectorInput.java similarity index 97% rename from java/src/main/java/org/rocksdb/StatsCollectorInput.java rename to java/src/main/java/org/forstdb/StatsCollectorInput.java index 5bf43ade5..331957064 100644 --- a/java/src/main/java/org/rocksdb/StatsCollectorInput.java +++ b/java/src/main/java/org/forstdb/StatsCollectorInput.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Contains all information necessary to collect statistics from one instance diff --git a/java/src/main/java/org/rocksdb/StatsLevel.java b/java/src/main/java/org/forstdb/StatsLevel.java similarity index 95% rename from java/src/main/java/org/rocksdb/StatsLevel.java rename to java/src/main/java/org/forstdb/StatsLevel.java index 8190e503a..8ce3910bf 100644 --- a/java/src/main/java/org/rocksdb/StatsLevel.java +++ b/java/src/main/java/org/forstdb/StatsLevel.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The level of Statistics to report. @@ -49,7 +49,7 @@ public byte getValue() { * * @param value byte representation of StatsLevel. * - * @return {@link org.rocksdb.StatsLevel} instance. + * @return {@link org.forstdb.StatsLevel} instance. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/Status.java b/java/src/main/java/org/forstdb/Status.java similarity index 96% rename from java/src/main/java/org/rocksdb/Status.java rename to java/src/main/java/org/forstdb/Status.java index 5c50e700f..db7223ee3 100644 --- a/java/src/main/java/org/rocksdb/Status.java +++ b/java/src/main/java/org/forstdb/Status.java @@ -3,8 +3,9 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; +import java.io.Serializable; import java.util.Objects; /** @@ -13,7 +14,8 @@ * Currently only used with {@link RocksDBException} when the * status is not {@link Code#Ok} */ -public class Status { +public class Status implements Serializable { + private static final long serialVersionUID = -3794191127754280439L; private final Code code; /* @Nullable */ private final SubCode subCode; /* @Nullable */ private final String state; diff --git a/java/src/main/java/org/rocksdb/StringAppendOperator.java b/java/src/main/java/org/forstdb/StringAppendOperator.java similarity index 97% rename from java/src/main/java/org/rocksdb/StringAppendOperator.java rename to java/src/main/java/org/forstdb/StringAppendOperator.java index 547371e7c..befb215c1 100644 --- a/java/src/main/java/org/rocksdb/StringAppendOperator.java +++ b/java/src/main/java/org/forstdb/StringAppendOperator.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * StringAppendOperator is a merge operator that concatenates diff --git a/java/src/main/java/org/rocksdb/TableFileCreationBriefInfo.java b/java/src/main/java/org/forstdb/TableFileCreationBriefInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/TableFileCreationBriefInfo.java rename to java/src/main/java/org/forstdb/TableFileCreationBriefInfo.java index 8dc56796a..5246a868c 100644 --- a/java/src/main/java/org/rocksdb/TableFileCreationBriefInfo.java +++ b/java/src/main/java/org/forstdb/TableFileCreationBriefInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/TableFileCreationInfo.java b/java/src/main/java/org/forstdb/TableFileCreationInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/TableFileCreationInfo.java rename to java/src/main/java/org/forstdb/TableFileCreationInfo.java index 5654603c3..f9c3c368e 100644 --- a/java/src/main/java/org/rocksdb/TableFileCreationInfo.java +++ b/java/src/main/java/org/forstdb/TableFileCreationInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/TableFileCreationReason.java b/java/src/main/java/org/forstdb/TableFileCreationReason.java similarity index 98% rename from java/src/main/java/org/rocksdb/TableFileCreationReason.java rename to java/src/main/java/org/forstdb/TableFileCreationReason.java index d3984663d..13cfb832f 100644 --- a/java/src/main/java/org/rocksdb/TableFileCreationReason.java +++ b/java/src/main/java/org/forstdb/TableFileCreationReason.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum TableFileCreationReason { FLUSH((byte) 0x00), diff --git a/java/src/main/java/org/rocksdb/TableFileDeletionInfo.java b/java/src/main/java/org/forstdb/TableFileDeletionInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/TableFileDeletionInfo.java rename to java/src/main/java/org/forstdb/TableFileDeletionInfo.java index 9a777e333..61a3fdba6 100644 --- a/java/src/main/java/org/rocksdb/TableFileDeletionInfo.java +++ b/java/src/main/java/org/forstdb/TableFileDeletionInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/rocksdb/TableFilter.java b/java/src/main/java/org/forstdb/TableFilter.java similarity index 97% rename from java/src/main/java/org/rocksdb/TableFilter.java rename to java/src/main/java/org/forstdb/TableFilter.java index a39a329fb..0b4e8b400 100644 --- a/java/src/main/java/org/rocksdb/TableFilter.java +++ b/java/src/main/java/org/forstdb/TableFilter.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** * Filter for iterating a table. diff --git a/java/src/main/java/org/rocksdb/TableFormatConfig.java b/java/src/main/java/org/forstdb/TableFormatConfig.java similarity index 97% rename from java/src/main/java/org/rocksdb/TableFormatConfig.java rename to java/src/main/java/org/forstdb/TableFormatConfig.java index 726c6f122..891b3cb72 100644 --- a/java/src/main/java/org/rocksdb/TableFormatConfig.java +++ b/java/src/main/java/org/forstdb/TableFormatConfig.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * TableFormatConfig is used to config the internal Table format of a RocksDB. diff --git a/java/src/main/java/org/rocksdb/TableProperties.java b/java/src/main/java/org/forstdb/TableProperties.java similarity index 98% rename from java/src/main/java/org/rocksdb/TableProperties.java rename to java/src/main/java/org/forstdb/TableProperties.java index 02b95608e..4243ce9fe 100644 --- a/java/src/main/java/org/rocksdb/TableProperties.java +++ b/java/src/main/java/org/forstdb/TableProperties.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import java.util.Arrays; import java.util.Map; @@ -46,6 +46,7 @@ public class TableProperties { * Access is package private as this will only be constructed from * C++ via JNI and for testing. */ + @SuppressWarnings("PMD.ArrayIsStoredDirectly") TableProperties(final long dataSize, final long indexSize, final long indexPartitions, final long topLevelIndexSize, final long indexKeyIsUserKey, final long indexValueIsDeltaEncoded, final long filterSize, final long rawKeySize, @@ -116,6 +117,7 @@ public long getIndexSize() { * * @return the total number of index partitions. */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public long getIndexPartitions() { return indexPartitions; } @@ -299,6 +301,7 @@ public long getFastCompressionEstimatedDataSize() { * @return the name of the column family, or null if the * column family is unknown. */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") /*@Nullable*/ public byte[] getColumnFamilyName() { return columnFamilyName; } diff --git a/java/src/main/java/org/rocksdb/ThreadStatus.java b/java/src/main/java/org/forstdb/ThreadStatus.java similarity index 98% rename from java/src/main/java/org/rocksdb/ThreadStatus.java rename to java/src/main/java/org/forstdb/ThreadStatus.java index 38e7fad9c..f1a9e5c98 100644 --- a/java/src/main/java/org/rocksdb/ThreadStatus.java +++ b/java/src/main/java/org/forstdb/ThreadStatus.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Map; @@ -118,6 +118,7 @@ public OperationStage getOperationStage() { * * @return the properties */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public long[] getOperationProperties() { return operationProperties; } diff --git a/java/src/main/java/org/rocksdb/ThreadType.java b/java/src/main/java/org/forstdb/ThreadType.java similarity index 98% rename from java/src/main/java/org/rocksdb/ThreadType.java rename to java/src/main/java/org/forstdb/ThreadType.java index cc329f442..4f324c338 100644 --- a/java/src/main/java/org/rocksdb/ThreadType.java +++ b/java/src/main/java/org/forstdb/ThreadType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The type of a thread. diff --git a/java/src/main/java/org/rocksdb/TickerType.java b/java/src/main/java/org/forstdb/TickerType.java similarity index 97% rename from java/src/main/java/org/rocksdb/TickerType.java rename to java/src/main/java/org/forstdb/TickerType.java index c167f74c4..aef29e31a 100644 --- a/java/src/main/java/org/rocksdb/TickerType.java +++ b/java/src/main/java/org/forstdb/TickerType.java @@ -3,16 +3,16 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The logical mapping of tickers defined in rocksdb::Tickers. *

    * Java byte value mappings don't align 1:1 to the c++ values. c++ rocksdb::Tickers enumeration type - * is uint32_t and java org.rocksdb.TickerType is byte, this causes mapping issues when + * is uint32_t and java org.forstdb.TickerType is byte, this causes mapping issues when * rocksdb::Tickers value is greater then 127 (0x7F) for jbyte jni interface as range greater is not * available. Without breaking interface in minor versions, value mappings for - * org.rocksdb.TickerType leverage full byte range [-128 (-0x80), (0x7F)]. Newer tickers added + * org.forstdb.TickerType leverage full byte range [-128 (-0x80), (0x7F)]. Newer tickers added * should descend into negative values until TICKER_ENUM_MAX reaches -128 (-0x80). */ public enum TickerType { @@ -764,6 +764,18 @@ public enum TickerType { */ BLOCK_CHECKSUM_MISMATCH_COUNT((byte) -0x3C), + READAHEAD_TRIMMED((byte) -0x3D), + + FIFO_MAX_SIZE_COMPACTIONS((byte) -0x3E), + + FIFO_TTL_COMPACTIONS((byte) -0x3F), + + PREFETCH_BYTES((byte) -0x40), + + PREFETCH_BYTES_USEFUL((byte) -0x41), + + PREFETCH_HITS((byte) -0x42), + TICKER_ENUM_MAX((byte) 0x5F); private final byte value; @@ -786,7 +798,7 @@ public byte getValue() { * * @param value byte representation of TickerType. * - * @return {@link org.rocksdb.TickerType} instance. + * @return {@link org.forstdb.TickerType} instance. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ diff --git a/java/src/main/java/org/rocksdb/TimedEnv.java b/java/src/main/java/org/forstdb/TimedEnv.java similarity index 97% rename from java/src/main/java/org/rocksdb/TimedEnv.java rename to java/src/main/java/org/forstdb/TimedEnv.java index dc8b5d6ef..d8d703db7 100644 --- a/java/src/main/java/org/rocksdb/TimedEnv.java +++ b/java/src/main/java/org/forstdb/TimedEnv.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Timed environment. diff --git a/java/src/main/java/org/rocksdb/TraceOptions.java b/java/src/main/java/org/forstdb/TraceOptions.java similarity index 97% rename from java/src/main/java/org/rocksdb/TraceOptions.java rename to java/src/main/java/org/forstdb/TraceOptions.java index cf5f7bbe1..45eb80624 100644 --- a/java/src/main/java/org/rocksdb/TraceOptions.java +++ b/java/src/main/java/org/forstdb/TraceOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * TraceOptions is used for diff --git a/java/src/main/java/org/rocksdb/TraceWriter.java b/java/src/main/java/org/forstdb/TraceWriter.java similarity index 97% rename from java/src/main/java/org/rocksdb/TraceWriter.java rename to java/src/main/java/org/forstdb/TraceWriter.java index cb0234e9b..baaa640c9 100644 --- a/java/src/main/java/org/rocksdb/TraceWriter.java +++ b/java/src/main/java/org/forstdb/TraceWriter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * TraceWriter allows exporting RocksDB traces to any system, diff --git a/java/src/main/java/org/rocksdb/Transaction.java b/java/src/main/java/org/forstdb/Transaction.java similarity index 63% rename from java/src/main/java/org/rocksdb/Transaction.java rename to java/src/main/java/org/forstdb/Transaction.java index 7d61a208e..f5bc2de3c 100644 --- a/java/src/main/java/org/rocksdb/Transaction.java +++ b/java/src/main/java/org/forstdb/Transaction.java @@ -3,8 +3,11 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; +import static org.forstdb.RocksDB.PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD; + +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -16,8 +19,8 @@ * {@link OptimisticTransactionDB} or a {@link TransactionDB} * * To create a transaction, use - * {@link OptimisticTransactionDB#beginTransaction(org.rocksdb.WriteOptions)} or - * {@link TransactionDB#beginTransaction(org.rocksdb.WriteOptions)} + * {@link OptimisticTransactionDB#beginTransaction(org.forstdb.WriteOptions)} or + * {@link TransactionDB#beginTransaction(org.forstdb.WriteOptions)} * * It is up to the caller to synchronize access to this object. *

    @@ -26,14 +29,19 @@ * examples. */ public class Transaction extends RocksObject { + private static final String FOR_EACH_KEY_THERE_MUST_BE_A_COLUMNFAMILYHANDLE = + "For each key there must be a ColumnFamilyHandle."; + private static final String BB_ALL_DIRECT_OR_INDIRECT = + "ByteBuffer parameters must all be direct, or must all be indirect"; private final RocksDB parent; + private final ColumnFamilyHandle defaultColumnFamilyHandle; /** * Intentionally package private * as this is called from - * {@link OptimisticTransactionDB#beginTransaction(org.rocksdb.WriteOptions)} - * or {@link TransactionDB#beginTransaction(org.rocksdb.WriteOptions)} + * {@link OptimisticTransactionDB#beginTransaction(org.forstdb.WriteOptions)} + * or {@link TransactionDB#beginTransaction(org.forstdb.WriteOptions)} * * @param parent This must be either {@link TransactionDB} or * {@link OptimisticTransactionDB} @@ -43,6 +51,7 @@ public class Transaction extends RocksObject { Transaction(final RocksDB parent, final long transactionHandle) { super(transactionHandle); this.parent = parent; + this.defaultColumnFamilyHandle = parent.getDefaultColumnFamily(); } /** @@ -246,6 +255,9 @@ public void rollbackToSavePoint() throws RocksDBException { } /** + * This function has an inconsistent parameter order compared to other {@code get()} + * methods and is deprecated in favour of one with a consistent order. + * * This function is similar to * {@link RocksDB#get(ColumnFamilyHandle, ReadOptions, byte[])} except it will * also read pending changes in this transaction. @@ -261,7 +273,7 @@ public void rollbackToSavePoint() throws RocksDBException { * transaction (the keys in this transaction do not yet belong to any snapshot * and will be fetched regardless). * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} instance * @param readOptions Read options. * @param key the key to retrieve the value for. * @@ -271,10 +283,44 @@ public void rollbackToSavePoint() throws RocksDBException { * @throws RocksDBException thrown if error happens in underlying native * library. */ - public byte[] get(final ColumnFamilyHandle columnFamilyHandle, - final ReadOptions readOptions, final byte[] key) throws RocksDBException { - assert(isOwningHandle()); - return get(nativeHandle_, readOptions.nativeHandle_, key, key.length, + @Deprecated + public byte[] get(final ColumnFamilyHandle columnFamilyHandle, final ReadOptions readOptions, + final byte[] key) throws RocksDBException { + assert (isOwningHandle()); + return get(nativeHandle_, readOptions.nativeHandle_, key, 0, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * This function is similar to + * {@link RocksDB#get(ColumnFamilyHandle, ReadOptions, byte[])} except it will + * also read pending changes in this transaction. + * Currently, this function will return Status::MergeInProgress if the most + * recent write to the queried key in this batch is a Merge. + * + * If {@link ReadOptions#snapshot()} is not set, the current version of the + * key will be read. Calling {@link #setSnapshot()} does not affect the + * version of the data returned. + * + * Note that setting {@link ReadOptions#setSnapshot(Snapshot)} will affect + * what is read from the DB but will NOT change which keys are read from this + * transaction (the keys in this transaction do not yet belong to any snapshot + * and will be fetched regardless). + * + * @param readOptions Read options. + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} instance + * @param key the key to retrieve the value for. + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying native + * library. + */ + public byte[] get(final ReadOptions readOptions, final ColumnFamilyHandle columnFamilyHandle, + final byte[] key) throws RocksDBException { + assert (isOwningHandle()); + return get(nativeHandle_, readOptions.nativeHandle_, key, 0, key.length, columnFamilyHandle.nativeHandle_); } @@ -306,7 +352,141 @@ public byte[] get(final ColumnFamilyHandle columnFamilyHandle, public byte[] get(final ReadOptions readOptions, final byte[] key) throws RocksDBException { assert(isOwningHandle()); - return get(nativeHandle_, readOptions.nativeHandle_, key, key.length); + return get(nativeHandle_, readOptions.nativeHandle_, key, 0, key.length, + defaultColumnFamilyHandle.nativeHandle_); + } + + /** + * Get the value associated with the specified key in the default column family + * + * @param opt {@link org.forstdb.ReadOptions} instance. + * @param key the key to retrieve the value. + * @param value the out-value to receive the retrieved value. + * @return A {@link GetStatus} wrapping the result status and the return value size. + * If {@code GetStatus.status} is {@code Ok} then {@code GetStatus.requiredSize} contains + * the size of the actual value that matches the specified + * {@code key} in byte. If {@code GetStatus.requiredSize} is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and a partial result was + * returned. If {@code GetStatus.status} is {@code NotFound} this indicates that + * the value was not found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public GetStatus get(final ReadOptions opt, final byte[] key, final byte[] value) + throws RocksDBException { + final int result = get(nativeHandle_, opt.nativeHandle_, key, 0, key.length, value, 0, + value.length, defaultColumnFamilyHandle.nativeHandle_); + if (result < 0) { + return GetStatus.fromStatusCode(Status.Code.NotFound, 0); + } else { + return GetStatus.fromStatusCode(Status.Code.Ok, result); + } + } + + /** + * Get the value associated with the specified key in a specified column family + * + * @param opt {@link org.forstdb.ReadOptions} instance. + * @param columnFamilyHandle the column family to find the key in + * @param key the key to retrieve the value. + * @param value the out-value to receive the retrieved value. + * @return A {@link GetStatus} wrapping the result status and the return value size. + * If {@code GetStatus.status} is {@code Ok} then {@code GetStatus.requiredSize} contains + * the size of the actual value that matches the specified + * {@code key} in byte. If {@code GetStatus.requiredSize} is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and a partial result was + * returned. If {@code GetStatus.status} is {@code NotFound} this indicates that + * the value was not found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public GetStatus get(final ReadOptions opt, final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final byte[] value) throws RocksDBException { + final int result = get(nativeHandle_, opt.nativeHandle_, key, 0, key.length, value, 0, + value.length, columnFamilyHandle.nativeHandle_); + if (result < 0) { + return GetStatus.fromStatusCode(Status.Code.NotFound, 0); + } else { + return GetStatus.fromStatusCode(Status.Code.Ok, result); + } + } + + /** + * Get the value associated with the specified key within the specified column family. + * + * @param opt {@link org.forstdb.ReadOptions} instance. + * @param columnFamilyHandle the column family in which to find the key. + * @param key the key to retrieve the value. It is using position and limit. + * Supports direct buffer only. + * @param value the out-value to receive the retrieved value. + * It is using position and limit. Limit is set according to value size. + * Supports direct buffer only. + * @return A {@link GetStatus} wrapping the result status and the return value size. + * If {@code GetStatus.status} is {@code Ok} then {@code GetStatus.requiredSize} contains + * the size of the actual value that matches the specified + * {@code key} in byte. If {@code GetStatus.requiredSize} is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and a partial result was + * returned. If {@code GetStatus.status} is {@code NotFound} this indicates that + * the value was not found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public GetStatus get(final ReadOptions opt, final ColumnFamilyHandle columnFamilyHandle, + final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + final int result; + if (key.isDirect() && value.isDirect()) { + result = getDirect(nativeHandle_, opt.nativeHandle_, key, key.position(), key.remaining(), + value, value.position(), value.remaining(), columnFamilyHandle.nativeHandle_); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + result = + get(nativeHandle_, opt.nativeHandle_, key.array(), key.arrayOffset() + key.position(), + key.remaining(), value.array(), value.arrayOffset() + value.position(), + value.remaining(), columnFamilyHandle.nativeHandle_); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + + key.position(key.limit()); + if (result < 0) { + return GetStatus.fromStatusCode(Status.Code.NotFound, 0); + } else { + value.position(Math.min(value.limit(), value.position() + result)); + return GetStatus.fromStatusCode(Status.Code.Ok, result); + } + } + + /** + * Get the value associated with the specified key within the default column family. + * + * @param opt {@link org.forstdb.ReadOptions} instance. + * @param key the key to retrieve the value. It is using position and limit. + * Supports direct buffer only. + * @param value the out-value to receive the retrieved value. + * It is using position and limit. Limit is set according to value size. + * Supports direct buffer only. + * @return A {@link GetStatus} wrapping the result status and the return value size. + * If {@code GetStatus.status} is {@code Ok} then {@code GetStatus.requiredSize} contains + * the size of the actual value that matches the specified + * {@code key} in byte. If {@code GetStatus.requiredSize} is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and a partial result was + * returned. If {@code GetStatus.status} is {@code NotFound} this indicates that + * the value was not found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public GetStatus get(final ReadOptions opt, final ByteBuffer key, final ByteBuffer value) + throws RocksDBException { + return get(opt, this.defaultColumnFamilyHandle, key, value); } /** @@ -327,7 +507,7 @@ public byte[] get(final ReadOptions readOptions, final byte[] key) * * @param readOptions Read options. * @param columnFamilyHandles {@link java.util.List} containing - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param keys of keys for which values need to be retrieved. * * @return Array of values, one for each key @@ -345,8 +525,7 @@ public byte[][] multiGet(final ReadOptions readOptions, // Check if key size equals cfList size. If not a exception must be // thrown. If not a Segmentation fault happens. if (keys.length != columnFamilyHandles.size()) { - throw new IllegalArgumentException( - "For each key there must be a ColumnFamilyHandle."); + throw new IllegalArgumentException(FOR_EACH_KEY_THERE_MUST_BE_A_COLUMNFAMILYHANDLE); } if(keys.length == 0) { return new byte[0][0]; @@ -378,7 +557,7 @@ public byte[][] multiGet(final ReadOptions readOptions, * * @param readOptions Read options. * @param columnFamilyHandles {@link java.util.List} containing - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param keys of keys for which values need to be retrieved. * * @return Array of values, one for each key @@ -396,9 +575,9 @@ public List multiGetAsList(final ReadOptions readOptions, // Check if key size equals cfList size. If not a exception must be // thrown. If not a Segmentation fault happens. if (keys.size() != columnFamilyHandles.size()) { - throw new IllegalArgumentException("For each key there must be a ColumnFamilyHandle."); + throw new IllegalArgumentException(FOR_EACH_KEY_THERE_MUST_BE_A_COLUMNFAMILYHANDLE); } - if (keys.size() == 0) { + if (keys.isEmpty()) { return new ArrayList<>(0); } final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); @@ -427,7 +606,7 @@ public List multiGetAsList(final ReadOptions readOptions, * and will be fetched regardless). * * @param readOptions Read options.= - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param keys of keys for which values need to be retrieved. * * @return Array of values, one for each key @@ -464,7 +643,7 @@ public byte[][] multiGet(final ReadOptions readOptions, final byte[][] keys) * and will be fetched regardless). * * @param readOptions Read options.= - * {@link org.rocksdb.ColumnFamilyHandle} instances. + * {@link org.forstdb.ColumnFamilyHandle} instances. * @param keys of keys for which values need to be retrieved. * * @return Array of values, one for each key @@ -474,7 +653,7 @@ public byte[][] multiGet(final ReadOptions readOptions, final byte[][] keys) */ public List multiGetAsList(final ReadOptions readOptions, final List keys) throws RocksDBException { - if (keys.size() == 0) { + if (keys.isEmpty()) { return new ArrayList<>(0); } final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); @@ -516,7 +695,7 @@ public List multiGetAsList(final ReadOptions readOptions, final List + * Note: Currently, this function will return Status::MergeInProgress + * if the most recent write to the queried key in this batch is a Merge. + *

    + * The values returned by this function are similar to + * {@link RocksDB#get(ReadOptions, byte[])}. + * If value==nullptr, then this function will not read any data, but will + * still ensure that this key cannot be written to by outside of this + * transaction. + *

    + * If this transaction was created on an {@link OptimisticTransactionDB}, + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)} + * could cause {@link #commit()} to fail. Otherwise, it could return any error + * that could be returned by + * {@link RocksDB#get(ReadOptions, byte[])}. + *

    + * If this transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * {@link Status.Code#MergeInProgress} if merge operations cannot be + * resolved. + * + * @param readOptions Read options. + * @param key the key to retrieve the value for. + * @param value the value associated with the input key if + * * any. The result is undefined in no value is associated with the key + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * + * @return a status object containing + * Status.OK if the requested value was read + * Status.NotFound if the requested value does not exist + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public GetStatus getForUpdate(final ReadOptions readOptions, final byte[] key, final byte[] value, + final boolean exclusive) throws RocksDBException { + final int result = getForUpdate(nativeHandle_, readOptions.nativeHandle_, key, 0, key.length, + value, 0, value.length, defaultColumnFamilyHandle.nativeHandle_, exclusive, + true /* doValidate */); + if (result < 0) { + return GetStatus.fromStatusCode(Status.Code.NotFound, 0); + } else { + return GetStatus.fromStatusCode(Status.Code.Ok, result); + } + } + + /** + * Read this key and ensure that this transaction will only + * be able to be committed if this key is not written outside this + * transaction after it has first been read (or after the snapshot if a + * snapshot is set in this transaction). The transaction behavior is the + * same regardless of whether the key exists or not. + *

    + * Note: Currently, this function will return Status::MergeInProgress + * if the most recent write to the queried key in this batch is a Merge. + *

    + * The values returned by this function are similar to + * {@link RocksDB#get(ReadOptions, byte[])}. + * If value==nullptr, then this function will not read any data, but will + * still ensure that this key cannot be written to by outside of this + * transaction. + *

    + * If this transaction was created on an {@link OptimisticTransactionDB}, + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)} + * could cause {@link #commit()} to fail. Otherwise, it could return any error + * that could be returned by + * {@link RocksDB#get(ReadOptions, byte[])}. + *

    + * If this transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * {@link Status.Code#MergeInProgress} if merge operations cannot be + * resolved. + * + * @param readOptions Read options. + * @param key the key to retrieve the value for. + * @param value the value associated with the input key if + * * any. The result is undefined in no value is associated with the key + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * + * @return a status object containing + * Status.OK if the requested value was read + * Status.NotFound if the requested value does not exist + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public GetStatus getForUpdate(final ReadOptions readOptions, final ByteBuffer key, + final ByteBuffer value, final boolean exclusive) throws RocksDBException { + return getForUpdate( + readOptions, defaultColumnFamilyHandle, key, value, exclusive, true /* doValidate */); + } + + /** + * Read this key and ensure that this transaction will only + * be able to be committed if this key is not written outside this + * transaction after it has first been read (or after the snapshot if a + * snapshot is set in this transaction). The transaction behavior is the + * same regardless of whether the key exists or not. + *

    + * Note: Currently, this function will return Status::MergeInProgress + * if the most recent write to the queried key in this batch is a Merge. + *

    + * The values returned by this function are similar to + * {@link RocksDB#get(ReadOptions, byte[])}. + * If value==nullptr, then this function will not read any data, but will + * still ensure that this key cannot be written to by outside of this + * transaction. + *

    + * If this transaction was created on an {@link OptimisticTransactionDB}, + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)} + * could cause {@link #commit()} to fail. Otherwise, it could return any error + * that could be returned by + * {@link RocksDB#get(ReadOptions, byte[])}. + *

    + * If this transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * {@link Status.Code#MergeInProgress} if merge operations cannot be + * resolved. + * + * @param readOptions Read options. + * @param columnFamilyHandle in which to find the key/value + * @param key the key to retrieve the value for. + * @param value the value associated with the input key if + * * any. The result is undefined in no value is associated with the key + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * + * @return a status object containing + * Status.OK if the requested value was read + * Status.NotFound if the requested value does not exist + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public GetStatus getForUpdate(final ReadOptions readOptions, + final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final byte[] value, + final boolean exclusive) throws RocksDBException { return getForUpdate( - nativeHandle_, readOptions.nativeHandle_, key, key.length, exclusive, true /*doValidate*/); + readOptions, columnFamilyHandle, key, value, exclusive, true /*doValidate*/); + } + + /** + * Read this key and ensure that this transaction will only + * be able to be committed if this key is not written outside this + * transaction after it has first been read (or after the snapshot if a + * snapshot is set in this transaction). The transaction behavior is the + * same regardless of whether the key exists or not. + *

    + * Note: Currently, this function will return Status::MergeInProgress + * if the most recent write to the queried key in this batch is a Merge. + *

    + * The values returned by this function are similar to + * {@link RocksDB#get(ReadOptions, byte[])}. + * If value==nullptr, then this function will not read any data, but will + * still ensure that this key cannot be written to by outside of this + * transaction. + *

    + * If this transaction was created on an {@link OptimisticTransactionDB}, + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)} + * could cause {@link #commit()} to fail. Otherwise, it could return any error + * that could be returned by + * {@link RocksDB#get(ReadOptions, byte[])}. + *

    + * If this transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * {@link Status.Code#MergeInProgress} if merge operations cannot be + * resolved. + * + * @param readOptions Read options. + * @param columnFamilyHandle in which to find the key/value + * @param key the key to retrieve the value for. + * @param value the value associated with the input key if + * * any. The result is undefined in no value is associated with the key + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * @param doValidate true if the transaction should validate the snapshot before doing the read + * + * @return a status object containing + * Status.OK if the requested value was read + * Status.NotFound if the requested value does not exist + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + + public GetStatus getForUpdate(final ReadOptions readOptions, + final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final byte[] value, + final boolean exclusive, final boolean doValidate) throws RocksDBException { + final int result = getForUpdate(nativeHandle_, readOptions.nativeHandle_, key, 0, key.length, + value, 0, value.length, columnFamilyHandle.nativeHandle_, exclusive, doValidate); + if (result < 0) { + return GetStatus.fromStatusCode(Status.Code.NotFound, 0); + } else { + return GetStatus.fromStatusCode(Status.Code.Ok, result); + } + } + + /** + * Read this key and ensure that this transaction will only + * be able to be committed if this key is not written outside this + * transaction after it has first been read (or after the snapshot if a + * snapshot is set in this transaction). The transaction behavior is the + * same regardless of whether the key exists or not. + *

    + * Note: Currently, this function will return Status::MergeInProgress + * if the most recent write to the queried key in this batch is a Merge. + *

    + * The values returned by this function are similar to + * {@link RocksDB#get(ReadOptions, byte[])}. + * If value==nullptr, then this function will not read any data, but will + * still ensure that this key cannot be written to by outside of this + * transaction. + *

    + * If this transaction was created on an {@link OptimisticTransactionDB}, + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)} + * could cause {@link #commit()} to fail. Otherwise, it could return any error + * that could be returned by + * {@link RocksDB#get(ReadOptions, byte[])}. + *

    + * If this transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * {@link Status.Code#MergeInProgress} if merge operations cannot be + * resolved. + * + * @param readOptions Read options. + * @param columnFamilyHandle in which to find the key/value + * @param key the key to retrieve the value for. + * @param value the value associated with the input key if + * * any. The result is undefined in no value is associated with the key + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * + * @return a status object containing + * Status.OK if the requested value was read + * Status.NotFound if the requested value does not exist + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + + public GetStatus getForUpdate(final ReadOptions readOptions, + final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, final ByteBuffer value, + final boolean exclusive) throws RocksDBException { + return getForUpdate( + readOptions, columnFamilyHandle, key, value, exclusive, true /*doValidate*/); + } + + /** + * Read this key and ensure that this transaction will only + * be able to be committed if this key is not written outside this + * transaction after it has first been read (or after the snapshot if a + * snapshot is set in this transaction). The transaction behavior is the + * same regardless of whether the key exists or not. + *

    + * Note: Currently, this function will return Status::MergeInProgress + * if the most recent write to the queried key in this batch is a Merge. + *

    + * The values returned by this function are similar to + * {@link RocksDB#get(ReadOptions, byte[])}. + * If value==nullptr, then this function will not read any data, but will + * still ensure that this key cannot be written to by outside of this + * transaction. + *

    + * If this transaction was created on an {@link OptimisticTransactionDB}, + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)} + * could cause {@link #commit()} to fail. Otherwise, it could return any error + * that could be returned by + * {@link RocksDB#get(ReadOptions, byte[])}. + *

    + * If this transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * {@link Status.Code#MergeInProgress} if merge operations cannot be + * resolved. + * + * @param readOptions Read options. + * @param columnFamilyHandle in which to find the key/value + * @param key the key to retrieve the value for. + * @param value the value associated with the input key if + * * any. The result is undefined in no value is associated with the key + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * @param doValidate true if the transaction should validate the snapshot before doing the read + * + * @return a status object containing + * Status.OK if the requested value was read + * Status.NotFound if the requested value does not exist + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public GetStatus getForUpdate(final ReadOptions readOptions, + final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, final ByteBuffer value, + final boolean exclusive, final boolean doValidate) throws RocksDBException { + final int result; + if (key.isDirect() && value.isDirect()) { + result = getDirectForUpdate(nativeHandle_, readOptions.nativeHandle_, key, key.position(), + key.remaining(), value, value.position(), value.remaining(), + columnFamilyHandle.nativeHandle_, exclusive, doValidate); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + result = getForUpdate(nativeHandle_, readOptions.nativeHandle_, key.array(), + key.arrayOffset() + key.position(), key.remaining(), value.array(), + value.arrayOffset() + value.position(), value.remaining(), + columnFamilyHandle.nativeHandle_, exclusive, doValidate); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + key.position(key.limit()); + if (result < 0) { + return GetStatus.fromStatusCode(Status.Code.NotFound, 0); + } else { + value.position(Math.min(value.limit(), value.position() + result)); + return GetStatus.fromStatusCode(Status.Code.Ok, result); + } } /** @@ -620,7 +1160,7 @@ public byte[] getForUpdate(final ReadOptions readOptions, final byte[] key, *

    * * @param readOptions Read options. - * @param columnFamilyHandles {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandles {@link org.forstdb.ColumnFamilyHandle} * instances * @param keys the keys to retrieve the values for. * @@ -637,8 +1177,7 @@ public byte[][] multiGetForUpdate(final ReadOptions readOptions, // Check if key size equals cfList size. If not a exception must be // thrown. If not a Segmentation fault happens. if (keys.length != columnFamilyHandles.size()){ - throw new IllegalArgumentException( - "For each key there must be a ColumnFamilyHandle."); + throw new IllegalArgumentException(FOR_EACH_KEY_THERE_MUST_BE_A_COLUMNFAMILYHANDLE); } if(keys.length == 0) { return new byte[0][0]; @@ -657,7 +1196,7 @@ public byte[][] multiGetForUpdate(final ReadOptions readOptions, *

    * * @param readOptions Read options. - * @param columnFamilyHandles {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandles {@link org.forstdb.ColumnFamilyHandle} * instances * @param keys the keys to retrieve the values for. * @@ -673,9 +1212,9 @@ public List multiGetForUpdateAsList(final ReadOptions readOptions, // Check if key size equals cfList size. If not a exception must be // thrown. If not a Segmentation fault happens. if (keys.size() != columnFamilyHandles.size()) { - throw new IllegalArgumentException("For each key there must be a ColumnFamilyHandle."); + throw new IllegalArgumentException(FOR_EACH_KEY_THERE_MUST_BE_A_COLUMNFAMILYHANDLE); } - if (keys.size() == 0) { + if (keys.isEmpty()) { return new ArrayList<>(); } final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); @@ -727,7 +1266,7 @@ public byte[][] multiGetForUpdate(final ReadOptions readOptions, final byte[][] public List multiGetForUpdateAsList( final ReadOptions readOptions, final List keys) throws RocksDBException { assert (isOwningHandle()); - if (keys.size() == 0) { + if (keys.isEmpty()) { return new ArrayList<>(0); } @@ -740,7 +1279,28 @@ public List multiGetForUpdateAsList( * Returns an iterator that will iterate on all keys in the default * column family including both keys in the DB and uncommitted keys in this * transaction. - *

    + *

    + * Caller is responsible for deleting the returned Iterator. + *

    + * The returned iterator is only valid until {@link #commit()}, + * {@link #rollback()}, or {@link #rollbackToSavePoint()} is called. + * + * @return instance of iterator object. + */ + public RocksIterator getIterator() { + assert (isOwningHandle()); + try (ReadOptions readOptions = new ReadOptions()) { + return new RocksIterator(parent, + getIterator( + nativeHandle_, readOptions.nativeHandle_, defaultColumnFamilyHandle.nativeHandle_)); + } + } + + /** + * Returns an iterator that will iterate on all keys in the default + * column family including both keys in the DB and uncommitted keys in this + * transaction. + * * Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is read * from the DB but will NOT change which keys are read from this transaction * (the keys in this transaction do not yet belong to any snapshot and will be @@ -757,8 +1317,9 @@ public List multiGetForUpdateAsList( */ public RocksIterator getIterator(final ReadOptions readOptions) { assert(isOwningHandle()); - return new RocksIterator(parent, getIterator(nativeHandle_, - readOptions.nativeHandle_)); + return new RocksIterator(parent, + getIterator( + nativeHandle_, readOptions.nativeHandle_, defaultColumnFamilyHandle.nativeHandle_)); } /** @@ -778,7 +1339,7 @@ public RocksIterator getIterator(final ReadOptions readOptions) { * {@link #rollback()}, or {@link #rollbackToSavePoint()} is called. * * @param readOptions Read options. - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * * @return instance of iterator object. @@ -790,6 +1351,35 @@ public RocksIterator getIterator(final ReadOptions readOptions, readOptions.nativeHandle_, columnFamilyHandle.nativeHandle_)); } + /** + * Returns an iterator that will iterate on all keys in the column family + * specified by {@code columnFamilyHandle} including both keys in the DB + * and uncommitted keys in this transaction. + *

    + * Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is read + * from the DB but will NOT change which keys are read from this transaction + * (the keys in this transaction do not yet belong to any snapshot and will be + * fetched regardless). + *

    + * Caller is responsible for calling {@link RocksIterator#close()} on + * the returned Iterator. + *

    + * The returned iterator is only valid until {@link #commit()}, + * {@link #rollback()}, or {@link #rollbackToSavePoint()} is called. + * + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} + * instance + * + * @return instance of iterator object. + */ + public RocksIterator getIterator(final ColumnFamilyHandle columnFamilyHandle) { + assert (isOwningHandle()); + try (ReadOptions readOptions = new ReadOptions()) { + return new RocksIterator(parent, + getIterator(nativeHandle_, readOptions.nativeHandle_, columnFamilyHandle.nativeHandle_)); + } + } + /** * Similar to {@link RocksDB#put(ColumnFamilyHandle, byte[], byte[])}, but * will also perform conflict checking on the keys be written. @@ -821,8 +1411,8 @@ public RocksIterator getIterator(final ReadOptions readOptions, public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final byte[] value, final boolean assumeTracked) throws RocksDBException { assert (isOwningHandle()); - put(nativeHandle_, key, key.length, value, value.length, - columnFamilyHandle.nativeHandle_, assumeTracked); + put(nativeHandle_, key, 0, key.length, value, 0, value.length, columnFamilyHandle.nativeHandle_, + assumeTracked); } /** @@ -853,8 +1443,8 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final byte[] value) throws RocksDBException { assert(isOwningHandle()); - put(nativeHandle_, key, key.length, value, value.length, - columnFamilyHandle.nativeHandle_, false); + put(nativeHandle_, key, 0, key.length, value, 0, value.length, columnFamilyHandle.nativeHandle_, + false); } /** @@ -882,10 +1472,10 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, public void put(final byte[] key, final byte[] value) throws RocksDBException { assert(isOwningHandle()); - put(nativeHandle_, key, key.length, value, value.length); + put(nativeHandle_, key, 0, key.length, value, 0, value.length); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #put(ColumnFamilyHandle, byte[], byte[])} but allows * you to specify the key and value in several parts that will be @@ -933,7 +1523,97 @@ public void put(final ColumnFamilyHandle columnFamilyHandle, columnFamilyHandle.nativeHandle_, false); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link RocksDB#put(byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + * + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + * + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void put(final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + assert (isOwningHandle()); + if (key.isDirect() && value.isDirect()) { + putDirect(nativeHandle_, key, key.position(), key.remaining(), value, value.position(), + value.remaining()); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + put(nativeHandle_, key.array(), key.arrayOffset() + key.position(), key.remaining(), + value.array(), value.arrayOffset() + value.position(), value.remaining()); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + key.position(key.limit()); + value.position(value.limit()); + } + + /** + * Similar to {@link RocksDB#put(byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + * + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + * + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to put the key/value into + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * @param assumeTracked true when it is expected that the key is already + * tracked. More specifically, it means the the key was previous tracked + * in the same savepoint, with the same exclusive flag, and at a lower + * sequence number. If valid then it skips ValidateSnapshot, + * throws an error otherwise. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, + final ByteBuffer value, final boolean assumeTracked) throws RocksDBException { + assert (isOwningHandle()); + if (key.isDirect() && value.isDirect()) { + putDirect(nativeHandle_, key, key.position(), key.remaining(), value, value.position(), + value.remaining(), columnFamilyHandle.nativeHandle_, assumeTracked); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + put(nativeHandle_, key.array(), key.arrayOffset() + key.position(), key.remaining(), + value.array(), value.arrayOffset() + value.position(), value.remaining(), + columnFamilyHandle.nativeHandle_, assumeTracked); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + key.position(key.limit()); + value.position(value.limit()); + } + public void put(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, + final ByteBuffer value) throws RocksDBException { + put(columnFamilyHandle, key, value, false); + } + + // TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #put(byte[], byte[])} but allows * you to specify the key and value in several parts that will be @@ -984,7 +1664,7 @@ public void merge(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final byte[] value, final boolean assumeTracked) throws RocksDBException { assert (isOwningHandle()); - merge(nativeHandle_, key, key.length, value, value.length, + merge(nativeHandle_, key, 0, key.length, value, 0, value.length, columnFamilyHandle.nativeHandle_, assumeTracked); } @@ -1016,7 +1696,7 @@ public void merge(final ColumnFamilyHandle columnFamilyHandle, public void merge(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final byte[] value) throws RocksDBException { assert(isOwningHandle()); - merge(nativeHandle_, key, key.length, value, value.length, + merge(nativeHandle_, key, 0, key.length, value, 0, value.length, columnFamilyHandle.nativeHandle_, false); } @@ -1045,7 +1725,115 @@ public void merge(final ColumnFamilyHandle columnFamilyHandle, public void merge(final byte[] key, final byte[] value) throws RocksDBException { assert(isOwningHandle()); - merge(nativeHandle_, key, key.length, value, value.length); + merge(nativeHandle_, key, 0, key.length, value, 0, value.length); + } + + /** + * Similar to {@link RocksDB#merge(byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + * + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + * + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void merge(final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + assert (isOwningHandle()); + if (key.isDirect() && value.isDirect()) { + mergeDirect(nativeHandle_, key, key.position(), key.remaining(), value, value.position(), + value.remaining()); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + merge(nativeHandle_, key.array(), key.arrayOffset() + key.position(), key.remaining(), + value.array(), value.arrayOffset() + value.position(), value.remaining()); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + } + + /** + * Similar to {@link RocksDB#merge(byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + * + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + * + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle in which to apply the merge + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * @param assumeTracked expects the key be already tracked. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void merge(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, + final ByteBuffer value, final boolean assumeTracked) throws RocksDBException { + assert (isOwningHandle()); + if (key.isDirect() && value.isDirect()) { + mergeDirect(nativeHandle_, key, key.position(), key.remaining(), value, value.position(), + value.remaining(), columnFamilyHandle.nativeHandle_, assumeTracked); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + merge(nativeHandle_, key.array(), key.arrayOffset() + key.position(), key.remaining(), + value.array(), value.arrayOffset() + value.position(), value.remaining(), + columnFamilyHandle.nativeHandle_, assumeTracked); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + key.position(key.limit()); + value.position(value.limit()); + } + + /** + * Similar to {@link RocksDB#merge(byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + * + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + * + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle in which to apply the merge + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void merge(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, + final ByteBuffer value) throws RocksDBException { + merge(columnFamilyHandle, key, value, false); } /** @@ -1139,7 +1927,7 @@ public void delete(final byte[] key) throws RocksDBException { delete(nativeHandle_, key, key.length); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #delete(ColumnFamilyHandle, byte[])} but allows * you to specify the key in several parts that will be @@ -1184,7 +1972,7 @@ public void delete(final ColumnFamilyHandle columnFamilyHandle, columnFamilyHandle.nativeHandle_, false); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #delete(byte[])} but allows * you to specify key the in several parts that will be @@ -1227,9 +2015,9 @@ public void delete(final byte[][] keyParts) throws RocksDBException { * @throws RocksDBException when one of the TransactionalDB conditions * described above occurs, or in the case of an unexpected error */ - @Experimental("Performance optimization for a very specific workload") - public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, - final byte[] key, final boolean assumeTracked) throws RocksDBException { + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final boolean assumeTracked) throws RocksDBException { assert (isOwningHandle()); singleDelete(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_, assumeTracked); @@ -1259,9 +2047,9 @@ public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, * @throws RocksDBException when one of the TransactionalDB conditions * described above occurs, or in the case of an unexpected error */ - @Experimental("Performance optimization for a very specific workload") - public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, - final byte[] key) throws RocksDBException { + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, final byte[] key) + throws RocksDBException { assert(isOwningHandle()); singleDelete(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_, false); @@ -1288,13 +2076,13 @@ public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, * @throws RocksDBException when one of the TransactionalDB conditions * described above occurs, or in the case of an unexpected error */ - @Experimental("Performance optimization for a very specific workload") + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) public void singleDelete(final byte[] key) throws RocksDBException { assert(isOwningHandle()); singleDelete(nativeHandle_, key, key.length); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #singleDelete(ColumnFamilyHandle, byte[])} but allows * you to specify the key in several parts that will be @@ -1311,10 +2099,9 @@ public void singleDelete(final byte[] key) throws RocksDBException { * @throws RocksDBException when one of the TransactionalDB conditions * described above occurs, or in the case of an unexpected error */ - @Experimental("Performance optimization for a very specific workload") - public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, - final byte[][] keyParts, final boolean assumeTracked) - throws RocksDBException { + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, final byte[][] keyParts, + final boolean assumeTracked) throws RocksDBException { assert (isOwningHandle()); singleDelete(nativeHandle_, keyParts, keyParts.length, columnFamilyHandle.nativeHandle_, assumeTracked); @@ -1333,15 +2120,15 @@ public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, * @throws RocksDBException when one of the TransactionalDB conditions * described above occurs, or in the case of an unexpected error */ - @Experimental("Performance optimization for a very specific workload") - public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, - final byte[][] keyParts) throws RocksDBException { + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, final byte[][] keyParts) + throws RocksDBException { assert(isOwningHandle()); singleDelete(nativeHandle_, keyParts, keyParts.length, columnFamilyHandle.nativeHandle_, false); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #singleDelete(byte[])} but allows * you to specify the key in several parts that will be @@ -1352,7 +2139,7 @@ public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, * @throws RocksDBException when one of the TransactionalDB conditions * described above occurs, or in the case of an unexpected error */ - @Experimental("Performance optimization for a very specific workload") + @Experimental(PERFORMANCE_OPTIMIZATION_FOR_A_VERY_SPECIFIC_WORKLOAD) public void singleDelete(final byte[][] keyParts) throws RocksDBException { assert(isOwningHandle()); singleDelete(nativeHandle_, keyParts, keyParts.length); @@ -1410,7 +2197,7 @@ public void putUntracked(final byte[] key, final byte[] value) putUntracked(nativeHandle_, key, key.length, value, value.length); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #putUntracked(ColumnFamilyHandle, byte[], byte[])} but * allows you to specify the key and value in several parts that will be @@ -1431,7 +2218,7 @@ public void putUntracked(final ColumnFamilyHandle columnFamilyHandle, valueParts.length, columnFamilyHandle.nativeHandle_); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #putUntracked(byte[], byte[])} but * allows you to specify the key and value in several parts that will be @@ -1472,10 +2259,50 @@ public void putUntracked(final byte[][] keyParts, final byte[][] valueParts) */ public void mergeUntracked(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final byte[] value) throws RocksDBException { - mergeUntracked(nativeHandle_, key, key.length, value, value.length, + assert (isOwningHandle()); + mergeUntracked(nativeHandle_, key, 0, key.length, value, 0, value.length, columnFamilyHandle.nativeHandle_); } + /** + * Similar to {@link RocksDB#merge(ColumnFamilyHandle, byte[], byte[])}, + * but operates on the transactions write batch. This write will only happen + * if this transaction gets committed successfully. + * + * Unlike {@link #merge(ColumnFamilyHandle, byte[], byte[])} no conflict + * checking will be performed for this key. + * + * If this Transaction was created on a {@link TransactionDB}, this function + * will still acquire locks necessary to make sure this write doesn't cause + * conflicts in other transactions; This may cause a {@link RocksDBException} + * with associated {@link Status.Code#Busy}. + * + * @param columnFamilyHandle The column family to merge the key/value into + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void mergeUntracked(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, + final ByteBuffer value) throws RocksDBException { + assert (isOwningHandle()); + if (key.isDirect() && value.isDirect()) { + mergeUntrackedDirect(nativeHandle_, key, key.position(), key.remaining(), value, + value.position(), value.remaining(), columnFamilyHandle.nativeHandle_); + } else if (!key.isDirect() && !value.isDirect()) { + assert key.hasArray(); + assert value.hasArray(); + mergeUntracked(nativeHandle_, key.array(), key.arrayOffset() + key.position(), + key.remaining(), value.array(), value.arrayOffset() + value.position(), value.remaining(), + columnFamilyHandle.nativeHandle_); + } else { + throw new RocksDBException(BB_ALL_DIRECT_OR_INDIRECT); + } + key.position(key.limit()); + value.position(value.limit()); + } + /** * Similar to {@link RocksDB#merge(byte[], byte[])}, * but operates on the transactions write batch. This write will only happen @@ -1497,8 +2324,30 @@ public void mergeUntracked(final ColumnFamilyHandle columnFamilyHandle, */ public void mergeUntracked(final byte[] key, final byte[] value) throws RocksDBException { - assert(isOwningHandle()); - mergeUntracked(nativeHandle_, key, key.length, value, value.length); + mergeUntracked(defaultColumnFamilyHandle, key, value); + } + + /** + * Similar to {@link RocksDB#merge(byte[], byte[])}, + * but operates on the transactions write batch. This write will only happen + * if this transaction gets committed successfully. + * + * Unlike {@link #merge(byte[], byte[])} no conflict + * checking will be performed for this key. + * + * If this Transaction was created on a {@link TransactionDB}, this function + * will still acquire locks necessary to make sure this write doesn't cause + * conflicts in other transactions; This may cause a {@link RocksDBException} + * with associated {@link Status.Code#Busy}. + * + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void mergeUntracked(final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + mergeUntracked(defaultColumnFamilyHandle, key, value); } /** @@ -1550,7 +2399,7 @@ public void deleteUntracked(final byte[] key) throws RocksDBException { deleteUntracked(nativeHandle_, key, key.length); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #deleteUntracked(ColumnFamilyHandle, byte[])} but allows * you to specify the key in several parts that will be @@ -1569,7 +2418,7 @@ public void deleteUntracked(final ColumnFamilyHandle columnFamilyHandle, columnFamilyHandle.nativeHandle_); } - //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + //TODO(AR) refactor if we implement org.forstdb.SliceParts in future /** * Similar to {@link #deleteUntracked(byte[])} but allows * you to specify the key in several parts that will be @@ -1757,7 +2606,7 @@ public void setWriteOptions(final WriteOptions writeOptions) { * calling {@code #undoGetForUpdate(ColumnFamilyHandle, byte[])} may release * any held locks for this key. * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the key to retrieve the value for. */ @@ -1955,7 +2804,7 @@ public enum TransactionState { * * @param value byte representation of TransactionState. * - * @return {@link org.rocksdb.Transaction.TransactionState} instance or null. + * @return {@link org.forstdb.Transaction.TransactionState} instance or null. * @throws java.lang.IllegalArgumentException if an invalid * value is provided. */ @@ -1980,9 +2829,9 @@ public static TransactionState getTransactionState(final byte value) { * * @return The waiting transactions */ + @SuppressWarnings("PMD.UnusedPrivateMethod") private WaitingTransactions newWaitingTransactions( - final long columnFamilyId, final String key, - final long[] transactionIds) { + final long columnFamilyId, final String key, final long[] transactionIds) { return new WaitingTransactions(columnFamilyId, key, transactionIds); } @@ -2021,6 +2870,7 @@ public String getKey() { * * @return The IDs of the waiting transactions */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public long[] getTransactionIds() { return transactionIds; } @@ -2039,48 +2889,68 @@ private native void setSnapshotOnNextOperation(final long handle, private native void rollbackToSavePoint(final long handle) throws RocksDBException; private native byte[] get(final long handle, final long readOptionsHandle, final byte[] key, - final int keyLength, final long columnFamilyHandle) throws RocksDBException; - private native byte[] get(final long handle, final long readOptionsHandle, final byte[] key, - final int keyLen) throws RocksDBException; - private native byte[][] multiGet(final long handle, - final long readOptionsHandle, final byte[][] keys, - final long[] columnFamilyHandles) throws RocksDBException; + final int keyOffset, final int keyLength, final long columnFamilyHandle) + throws RocksDBException; + private native int get(final long handle, final long readOptionsHandle, final byte[] key, + final int keyOffset, final int keyLen, final byte[] value, final int valueOffset, + final int valueLen, final long columnFamilyHandle) throws RocksDBException; + private native int getDirect(final long handle, final long readOptionsHandle, + final ByteBuffer key, final int keyOffset, final int keyLength, final ByteBuffer value, + final int valueOffset, final int valueLength, final long columnFamilyHandle) + throws RocksDBException; + + private native byte[][] multiGet(final long handle, final long readOptionsHandle, + final byte[][] keys, final long[] columnFamilyHandles) throws RocksDBException; private native byte[][] multiGet(final long handle, final long readOptionsHandle, final byte[][] keys) throws RocksDBException; private native byte[] getForUpdate(final long handle, final long readOptionsHandle, - final byte[] key, final int keyLength, final long columnFamilyHandle, final boolean exclusive, + final byte[] key, final int keyOffset, final int keyLength, final long columnFamilyHandle, + final boolean exclusive, final boolean doValidate) throws RocksDBException; + private native int getForUpdate(final long handle, final long readOptionsHandle, final byte[] key, + final int keyOffset, final int keyLength, final byte[] value, final int valueOffset, + final int valueLen, final long columnFamilyHandle, final boolean exclusive, final boolean doValidate) throws RocksDBException; - private native byte[] getForUpdate(final long handle, final long readOptionsHandle, - final byte[] key, final int keyLen, final boolean exclusive, final boolean doValidate) - throws RocksDBException; + private native int getDirectForUpdate(final long handle, final long readOptionsHandle, + final ByteBuffer key, final int keyOffset, final int keyLength, final ByteBuffer value, + final int valueOffset, final int valueLen, final long columnFamilyHandle, + final boolean exclusive, final boolean doValidate) throws RocksDBException; private native byte[][] multiGetForUpdate(final long handle, final long readOptionsHandle, final byte[][] keys, final long[] columnFamilyHandles) throws RocksDBException; - private native byte[][] multiGetForUpdate(final long handle, - final long readOptionsHandle, final byte[][] keys) - throws RocksDBException; - private native long getIterator(final long handle, - final long readOptionsHandle); + private native byte[][] multiGetForUpdate( + final long handle, final long readOptionsHandle, final byte[][] keys) throws RocksDBException; private native long getIterator(final long handle, final long readOptionsHandle, final long columnFamilyHandle); - private native void put(final long handle, final byte[] key, final int keyLength, - final byte[] value, final int valueLength, final long columnFamilyHandle, - final boolean assumeTracked) throws RocksDBException; - private native void put(final long handle, final byte[] key, - final int keyLength, final byte[] value, final int valueLength) + private native void put(final long handle, final byte[] key, final int keyOffset, + final int keyLength, final byte[] value, final int valueOffset, final int valueLength) throws RocksDBException; + private native void put(final long handle, final byte[] key, final int keyOffset, + final int keyLength, final byte[] value, final int valueOffset, final int valueLength, + final long columnFamilyHandle, final boolean assumeTracked) throws RocksDBException; private native void put(final long handle, final byte[][] keys, final int keysLength, final byte[][] values, final int valuesLength, final long columnFamilyHandle, final boolean assumeTracked) throws RocksDBException; private native void put(final long handle, final byte[][] keys, final int keysLength, final byte[][] values, final int valuesLength) throws RocksDBException; - private native void merge(final long handle, final byte[] key, final int keyLength, - final byte[] value, final int valueLength, final long columnFamilyHandle, + private native void putDirect(long handle, ByteBuffer key, int keyOffset, int keyLength, + ByteBuffer value, int valueOffset, int valueLength, long cfHandle, final boolean assumeTracked) throws RocksDBException; - private native void merge(final long handle, final byte[] key, - final int keyLength, final byte[] value, final int valueLength) + private native void putDirect(long handle, ByteBuffer key, int keyOffset, int keyLength, + ByteBuffer value, int valueOffset, int valueLength) throws RocksDBException; + + private native void merge(final long handle, final byte[] key, final int keyOffset, + final int keyLength, final byte[] value, final int valueOffset, final int valueLength, + final long columnFamilyHandle, final boolean assumeTracked) throws RocksDBException; + private native void mergeDirect(long handle, ByteBuffer key, int keyOffset, int keyLength, + ByteBuffer value, int valueOffset, int valueLength, long cfHandle, boolean assumeTracked) + throws RocksDBException; + private native void mergeDirect(long handle, ByteBuffer key, int keyOffset, int keyLength, + ByteBuffer value, int valueOffset, int valueLength) throws RocksDBException; + + private native void merge(final long handle, final byte[] key, final int keyOffset, + final int keyLength, final byte[] value, final int valueOffset, final int valueLength) throws RocksDBException; private native void delete(final long handle, final byte[] key, final int keyLength, final long columnFamilyHandle, final boolean assumeTracked) throws RocksDBException; @@ -2110,15 +2980,14 @@ private native void putUntracked(final long handle, final byte[][] keys, private native void putUntracked(final long handle, final byte[][] keys, final int keysLength, final byte[][] values, final int valuesLength) throws RocksDBException; - private native void mergeUntracked(final long handle, final byte[] key, - final int keyLength, final byte[] value, final int valueLength, + private native void mergeUntracked(final long handle, final byte[] key, final int keyOff, + final int keyLength, final byte[] value, final int valueOff, final int valueLength, + final long columnFamilyHandle) throws RocksDBException; + private native void mergeUntrackedDirect(final long handle, final ByteBuffer key, + final int keyOff, final int keyLength, final ByteBuffer value, final int valueOff, + final int valueLength, final long columnFamilyHandle) throws RocksDBException; + private native void deleteUntracked(final long handle, final byte[] key, final int keyLength, final long columnFamilyHandle) throws RocksDBException; - private native void mergeUntracked(final long handle, final byte[] key, - final int keyLength, final byte[] value, final int valueLength) - throws RocksDBException; - private native void deleteUntracked(final long handle, final byte[] key, - final int keyLength, final long columnFamilyHandle) - throws RocksDBException; private native void deleteUntracked(final long handle, final byte[] key, final int keyLength) throws RocksDBException; private native void deleteUntracked(final long handle, final byte[][] keys, diff --git a/java/src/main/java/org/rocksdb/TransactionDB.java b/java/src/main/java/org/forstdb/TransactionDB.java similarity index 93% rename from java/src/main/java/org/rocksdb/TransactionDB.java rename to java/src/main/java/org/forstdb/TransactionDB.java index 105f4eff0..ef4b36c7f 100644 --- a/java/src/main/java/org/rocksdb/TransactionDB.java +++ b/java/src/main/java/org/forstdb/TransactionDB.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.ArrayList; import java.util.List; @@ -14,8 +14,8 @@ */ public class TransactionDB extends RocksDB implements TransactionalDB { - - private TransactionDBOptions transactionDbOptions_; + // Field is "used" to prevent GC of the + @SuppressWarnings("PMD.UnusedPrivateField") private TransactionDBOptions transactionDbOptions_; /** * Private constructor. @@ -29,8 +29,8 @@ private TransactionDB(final long nativeHandle) { /** * Open a TransactionDB, similar to {@link RocksDB#open(Options, String)}. * - * @param options {@link org.rocksdb.Options} instance. - * @param transactionDbOptions {@link org.rocksdb.TransactionDBOptions} + * @param options {@link org.forstdb.Options} instance. + * @param transactionDbOptions {@link org.forstdb.TransactionDBOptions} * instance. * @param path the path to the rocksdb. * @@ -50,6 +50,7 @@ public static TransactionDB open(final Options options, // the currently-created RocksDB. tdb.storeOptionsInstance(options); tdb.storeTransactionDbOptions(transactionDbOptions); + tdb.storeDefaultColumnFamilyHandle(tdb.makeDefaultColumnFamilyHandle()); return tdb; } @@ -58,8 +59,8 @@ public static TransactionDB open(final Options options, * Open a TransactionDB, similar to * {@link RocksDB#open(DBOptions, String, List, List)}. * - * @param dbOptions {@link org.rocksdb.DBOptions} instance. - * @param transactionDbOptions {@link org.rocksdb.TransactionDBOptions} + * @param dbOptions {@link org.forstdb.DBOptions} instance. + * @param transactionDbOptions {@link org.forstdb.TransactionDBOptions} * instance. * @param path the path to the rocksdb. * @param columnFamilyDescriptors list of column family descriptors @@ -94,6 +95,7 @@ public static TransactionDB open(final DBOptions dbOptions, // in RocksDB can prevent Java to GC during the life-time of // the currently-created RocksDB. tdb.storeOptionsInstance(dbOptions); + tdb.storeDefaultColumnFamilyHandle(tdb.makeDefaultColumnFamilyHandle()); tdb.storeTransactionDbOptions(transactionDbOptions); for (int i = 1; i < handles.length; i++) { @@ -116,6 +118,7 @@ public static TransactionDB open(final DBOptions dbOptions, * * @throws RocksDBException if an error occurs whilst closing. */ + @Override public void closeE() throws RocksDBException { if (owningHandle_.compareAndSet(true, false)) { try { @@ -137,6 +140,7 @@ public void closeE() throws RocksDBException { *

    * See also {@link #close()}. */ + @SuppressWarnings("PMD.EmptyCatchBlock") @Override public void close() { if (owningHandle_.compareAndSet(true, false)) { @@ -218,7 +222,7 @@ public List getAllPreparedTransactions() { final List txns = new ArrayList<>(); for(final long jtxnHandle : jtxnHandles) { - final Transaction txn = new Transaction(this, jtxnHandle); + final Transaction txn = new Transaction(this, jtxnHandle); // NOPMD - CloseResource // this instance doesn't own the underlying C++ object txn.disOwnNativeHandle(); @@ -233,6 +237,7 @@ public static class KeyLockInfo { private final long[] transactionIDs; private final boolean exclusive; + @SuppressWarnings("PMD.ArrayIsStoredDirectly") public KeyLockInfo(final String key, final long[] transactionIDs, final boolean exclusive) { this.key = key; this.transactionIDs = transactionIDs; @@ -253,6 +258,7 @@ public String getKey() { * * @return the Transaction IDs. */ + @SuppressWarnings("PMD.MethodReturnsInternalArray") public long[] getTransactionIDs() { return transactionIDs; } @@ -287,8 +293,8 @@ public Map getLockStatusData() { * * @return The waiting transactions */ - private DeadlockInfo newDeadlockInfo( - final long transactionID, final long columnFamilyId, + @SuppressWarnings("PMD.UnusedPrivateMethod") + private DeadlockInfo newDeadlockInfo(final long transactionID, final long columnFamilyId, final String waitingKey, final boolean exclusive) { return new DeadlockInfo(transactionID, columnFamilyId, waitingKey, exclusive); @@ -349,6 +355,7 @@ public static class DeadlockPath { final DeadlockInfo[] path; final boolean limitExceeded; + @SuppressWarnings("PMD.ArrayIsStoredDirectly") public DeadlockPath(final DeadlockInfo[] path, final boolean limitExceeded) { this.path = path; this.limitExceeded = limitExceeded; diff --git a/java/src/main/java/org/rocksdb/TransactionDBOptions.java b/java/src/main/java/org/forstdb/TransactionDBOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/TransactionDBOptions.java rename to java/src/main/java/org/forstdb/TransactionDBOptions.java index 391025d6a..0ee96e10a 100644 --- a/java/src/main/java/org/rocksdb/TransactionDBOptions.java +++ b/java/src/main/java/org/forstdb/TransactionDBOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class TransactionDBOptions extends RocksObject { diff --git a/java/src/main/java/org/rocksdb/TransactionLogIterator.java b/java/src/main/java/org/forstdb/TransactionLogIterator.java similarity index 89% rename from java/src/main/java/org/rocksdb/TransactionLogIterator.java rename to java/src/main/java/org/forstdb/TransactionLogIterator.java index 5d9ec58d7..078191ea2 100644 --- a/java/src/main/java/org/rocksdb/TransactionLogIterator.java +++ b/java/src/main/java/org/forstdb/TransactionLogIterator.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** *

    A TransactionLogIterator is used to iterate over the transactions in a db. @@ -30,7 +30,7 @@ public void next() { /** *

    Throws RocksDBException if something went wrong.

    * - * @throws org.rocksdb.RocksDBException if something went + * @throws org.forstdb.RocksDBException if something went * wrong in the underlying C++ code. */ public void status() throws RocksDBException { @@ -44,7 +44,7 @@ public void status() throws RocksDBException { * *

    ONLY use if Valid() is true and status() is OK.

    * - * @return {@link org.rocksdb.TransactionLogIterator.BatchResult} + * @return {@link org.forstdb.TransactionLogIterator.BatchResult} * instance. */ public BatchResult getBatch() { @@ -71,7 +71,7 @@ public static final class BatchResult { *

    Constructor of BatchResult class.

    * * @param sequenceNumber related to this BatchResult instance. - * @param nativeHandle to {@link org.rocksdb.WriteBatch} + * @param nativeHandle to {@link org.forstdb.WriteBatch} * native instance. */ public BatchResult(final long sequenceNumber, @@ -90,10 +90,10 @@ public long sequenceNumber() { } /** - *

    Return contained {@link org.rocksdb.WriteBatch} + *

    Return contained {@link org.forstdb.WriteBatch} * instance

    * - * @return {@link org.rocksdb.WriteBatch} instance. + * @return {@link org.forstdb.WriteBatch} instance. */ public WriteBatch writeBatch() { return writeBatch_; diff --git a/java/src/main/java/org/rocksdb/TransactionOptions.java b/java/src/main/java/org/forstdb/TransactionOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/TransactionOptions.java rename to java/src/main/java/org/forstdb/TransactionOptions.java index f93d3cb3c..311403053 100644 --- a/java/src/main/java/org/rocksdb/TransactionOptions.java +++ b/java/src/main/java/org/forstdb/TransactionOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class TransactionOptions extends RocksObject implements TransactionalOptions { diff --git a/java/src/main/java/org/rocksdb/TransactionalDB.java b/java/src/main/java/org/forstdb/TransactionalDB.java similarity index 99% rename from java/src/main/java/org/rocksdb/TransactionalDB.java rename to java/src/main/java/org/forstdb/TransactionalDB.java index 1ba955496..ccf34720b 100644 --- a/java/src/main/java/org/rocksdb/TransactionalDB.java +++ b/java/src/main/java/org/forstdb/TransactionalDB.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; interface TransactionalDB> extends AutoCloseable { /** diff --git a/java/src/main/java/org/rocksdb/TransactionalOptions.java b/java/src/main/java/org/forstdb/TransactionalOptions.java similarity index 97% rename from java/src/main/java/org/rocksdb/TransactionalOptions.java rename to java/src/main/java/org/forstdb/TransactionalOptions.java index 2175693fd..cd7173789 100644 --- a/java/src/main/java/org/rocksdb/TransactionalOptions.java +++ b/java/src/main/java/org/forstdb/TransactionalOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; interface TransactionalOptions> diff --git a/java/src/main/java/org/rocksdb/TtlDB.java b/java/src/main/java/org/forstdb/TtlDB.java similarity index 96% rename from java/src/main/java/org/rocksdb/TtlDB.java rename to java/src/main/java/org/forstdb/TtlDB.java index 2bb0c4333..83ed89ae0 100644 --- a/java/src/main/java/org/rocksdb/TtlDB.java +++ b/java/src/main/java/org/forstdb/TtlDB.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.List; @@ -55,7 +55,7 @@ public class TtlDB extends RocksDB { * *

    Database is opened in read-write mode without default TTL.

    * - * @param options {@link org.rocksdb.Options} instance. + * @param options {@link org.forstdb.Options} instance. * @param db_path path to database. * * @return TtlDB instance. @@ -71,7 +71,7 @@ public static TtlDB open(final Options options, final String db_path) /** *

    Opens a TtlDB.

    * - * @param options {@link org.rocksdb.Options} instance. + * @param options {@link org.forstdb.Options} instance. * @param db_path path to database. * @param ttl time to live for new entries. * @param readOnly boolean value indicating if database if db is @@ -90,7 +90,7 @@ public static TtlDB open(final Options options, final String db_path, /** *

    Opens a TtlDB.

    * - * @param options {@link org.rocksdb.Options} instance. + * @param options {@link org.forstdb.Options} instance. * @param db_path path to database. * @param columnFamilyDescriptors list of column family descriptors * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances @@ -154,6 +154,7 @@ public static TtlDB open(final DBOptions options, final String db_path, * * @throws RocksDBException if an error occurs whilst closing. */ + @Override public void closeE() throws RocksDBException { if (owningHandle_.compareAndSet(true, false)) { try { @@ -175,6 +176,7 @@ public void closeE() throws RocksDBException { *

    * See also {@link #close()}. */ + @SuppressWarnings("PMD.EmptyCatchBlock") @Override public void close() { if (owningHandle_.compareAndSet(true, false)) { @@ -199,7 +201,7 @@ public void close() { * @param columnFamilyDescriptor column family to be created. * @param ttl TTL to set for this column family. * - * @return {@link org.rocksdb.ColumnFamilyHandle} instance. + * @return {@link org.forstdb.ColumnFamilyHandle} instance. * * @throws RocksDBException thrown if error happens in underlying * native library. diff --git a/java/src/main/java/org/rocksdb/TxnDBWritePolicy.java b/java/src/main/java/org/forstdb/TxnDBWritePolicy.java similarity index 98% rename from java/src/main/java/org/rocksdb/TxnDBWritePolicy.java rename to java/src/main/java/org/forstdb/TxnDBWritePolicy.java index 28cb8556b..b51dd4103 100644 --- a/java/src/main/java/org/rocksdb/TxnDBWritePolicy.java +++ b/java/src/main/java/org/forstdb/TxnDBWritePolicy.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The transaction db write policy. diff --git a/java/src/main/java/org/rocksdb/UInt64AddOperator.java b/java/src/main/java/org/forstdb/UInt64AddOperator.java similarity index 96% rename from java/src/main/java/org/rocksdb/UInt64AddOperator.java rename to java/src/main/java/org/forstdb/UInt64AddOperator.java index 0cffdce8c..2435eafa7 100644 --- a/java/src/main/java/org/rocksdb/UInt64AddOperator.java +++ b/java/src/main/java/org/forstdb/UInt64AddOperator.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Uint64AddOperator is a merge operator that accumlates a long diff --git a/java/src/main/java/org/rocksdb/VectorMemTableConfig.java b/java/src/main/java/org/forstdb/VectorMemTableConfig.java similarity index 98% rename from java/src/main/java/org/rocksdb/VectorMemTableConfig.java rename to java/src/main/java/org/forstdb/VectorMemTableConfig.java index fb1e7a948..039c68a1b 100644 --- a/java/src/main/java/org/rocksdb/VectorMemTableConfig.java +++ b/java/src/main/java/org/forstdb/VectorMemTableConfig.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; /** * The config for vector memtable representation. diff --git a/java/src/main/java/org/rocksdb/WALRecoveryMode.java b/java/src/main/java/org/forstdb/WALRecoveryMode.java similarity index 99% rename from java/src/main/java/org/rocksdb/WALRecoveryMode.java rename to java/src/main/java/org/forstdb/WALRecoveryMode.java index b8c098f94..c82b741db 100644 --- a/java/src/main/java/org/rocksdb/WALRecoveryMode.java +++ b/java/src/main/java/org/forstdb/WALRecoveryMode.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * The WAL Recover Mode diff --git a/java/src/main/java/org/rocksdb/WBWIRocksIterator.java b/java/src/main/java/org/forstdb/WBWIRocksIterator.java similarity index 93% rename from java/src/main/java/org/rocksdb/WBWIRocksIterator.java rename to java/src/main/java/org/forstdb/WBWIRocksIterator.java index e0b99b1b5..1e44fadb8 100644 --- a/java/src/main/java/org/rocksdb/WBWIRocksIterator.java +++ b/java/src/main/java/org/forstdb/WBWIRocksIterator.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; @@ -101,11 +101,11 @@ public void close() { /** * Represents an entry returned by - * {@link org.rocksdb.WBWIRocksIterator#entry()} + * {@link org.forstdb.WBWIRocksIterator#entry()} * * It is worth noting that a WriteEntry with - * the type {@link org.rocksdb.WBWIRocksIterator.WriteType#DELETE} - * or {@link org.rocksdb.WBWIRocksIterator.WriteType#LOG} + * the type {@link org.forstdb.WBWIRocksIterator.WriteType#DELETE} + * or {@link org.forstdb.WBWIRocksIterator.WriteType#LOG} * will not have a value. */ public static class WriteEntry implements AutoCloseable { @@ -118,7 +118,7 @@ public static class WriteEntry implements AutoCloseable { * should only be instantiated in * this manner by the outer WBWIRocksIterator * class; The class members are then modified - * by calling {@link org.rocksdb.WBWIRocksIterator#entry()} + * by calling {@link org.forstdb.WBWIRocksIterator#entry()} */ private WriteEntry() { key = new DirectSlice(); @@ -159,10 +159,10 @@ public DirectSlice getKey() { * no value */ public DirectSlice getValue() { - if(!value.isOwningHandle()) { - return null; //TODO(AR) migrate to JDK8 java.util.Optional#empty() - } else { + if (value.isOwningHandle()) { return value; + } else { + return null; // TODO(AR) migrate to JDK8 java.util.Optional#empty() } } @@ -178,6 +178,7 @@ public int hashCode() { return (key == null) ? 0 : key.hashCode(); } + @SuppressWarnings("PMD.CloseResource") @Override public boolean equals(final Object other) { if(other == null) { diff --git a/java/src/main/java/org/rocksdb/WalFileType.java b/java/src/main/java/org/forstdb/WalFileType.java similarity index 98% rename from java/src/main/java/org/rocksdb/WalFileType.java rename to java/src/main/java/org/forstdb/WalFileType.java index fed27ed11..117c59aa4 100644 --- a/java/src/main/java/org/rocksdb/WalFileType.java +++ b/java/src/main/java/org/forstdb/WalFileType.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum WalFileType { /** diff --git a/java/src/main/java/org/rocksdb/WalFilter.java b/java/src/main/java/org/forstdb/WalFilter.java similarity index 99% rename from java/src/main/java/org/rocksdb/WalFilter.java rename to java/src/main/java/org/forstdb/WalFilter.java index a2836634a..330d20c88 100644 --- a/java/src/main/java/org/rocksdb/WalFilter.java +++ b/java/src/main/java/org/forstdb/WalFilter.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Map; diff --git a/java/src/main/java/org/rocksdb/WalProcessingOption.java b/java/src/main/java/org/forstdb/WalProcessingOption.java similarity index 98% rename from java/src/main/java/org/rocksdb/WalProcessingOption.java rename to java/src/main/java/org/forstdb/WalProcessingOption.java index 3a9c2be0e..36ee14c99 100644 --- a/java/src/main/java/org/rocksdb/WalProcessingOption.java +++ b/java/src/main/java/org/forstdb/WalProcessingOption.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum WalProcessingOption { /* diff --git a/java/src/main/java/org/rocksdb/WriteBatch.java b/java/src/main/java/org/forstdb/WriteBatch.java similarity index 99% rename from java/src/main/java/org/rocksdb/WriteBatch.java rename to java/src/main/java/org/forstdb/WriteBatch.java index 49e1f7f20..1cc7736ca 100644 --- a/java/src/main/java/org/rocksdb/WriteBatch.java +++ b/java/src/main/java/org/forstdb/WriteBatch.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; diff --git a/java/src/main/java/org/rocksdb/WriteBatchInterface.java b/java/src/main/java/org/forstdb/WriteBatchInterface.java similarity index 98% rename from java/src/main/java/org/rocksdb/WriteBatchInterface.java rename to java/src/main/java/org/forstdb/WriteBatchInterface.java index 32cd8d1e7..6c7166875 100644 --- a/java/src/main/java/org/rocksdb/WriteBatchInterface.java +++ b/java/src/main/java/org/forstdb/WriteBatchInterface.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; @@ -33,7 +33,7 @@ public interface WriteBatchInterface { *

    Store the mapping "key->value" within given column * family.

    * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the specified key to be inserted. * @param value the value associated with the specified key. @@ -58,7 +58,7 @@ void put(ColumnFamilyHandle columnFamilyHandle, byte[] key, byte[] value) *

    Store the mapping "key->value" within given column * family.

    * - * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * @param columnFamilyHandle {@link org.forstdb.ColumnFamilyHandle} * instance * @param key the specified key to be inserted. It is using position and limit. * Supports direct buffer only. diff --git a/java/src/main/java/org/rocksdb/WriteBatchWithIndex.java b/java/src/main/java/org/forstdb/WriteBatchWithIndex.java similarity index 94% rename from java/src/main/java/org/rocksdb/WriteBatchWithIndex.java rename to java/src/main/java/org/forstdb/WriteBatchWithIndex.java index d41be5856..02a3be8fb 100644 --- a/java/src/main/java/org/rocksdb/WriteBatchWithIndex.java +++ b/java/src/main/java/org/forstdb/WriteBatchWithIndex.java @@ -3,20 +3,20 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.ByteBuffer; /** - * Similar to {@link org.rocksdb.WriteBatch} but with a binary searchable + * Similar to {@link org.forstdb.WriteBatch} but with a binary searchable * index built for all the keys inserted. *

    * Calling put, merge, remove or putLogData calls the same function - * as with {@link org.rocksdb.WriteBatch} whilst also building an index. + * as with {@link org.forstdb.WriteBatch} whilst also building an index. *

    - * A user can call {@link org.rocksdb.WriteBatchWithIndex#newIterator()} to + * A user can call {@link org.forstdb.WriteBatchWithIndex#newIterator()} to * create an iterator over the write batch or - * {@link org.rocksdb.WriteBatchWithIndex#newIteratorWithBase(org.rocksdb.RocksIterator)} + * {@link org.forstdb.WriteBatchWithIndex#newIteratorWithBase(org.forstdb.RocksIterator)} * to get an iterator for the database with Read-Your-Own-Writes like capability */ public class WriteBatchWithIndex extends AbstractWriteBatch { @@ -80,7 +80,7 @@ public WriteBatchWithIndex( /** * Create an iterator of a column family. User can call - * {@link org.rocksdb.RocksIteratorInterface#seek(byte[])} to + * {@link org.forstdb.RocksIteratorInterface#seek(byte[])} to * search to the next entry of or after a key. Keys will be iterated in the * order given by index_comparator. For multiple updates on the same key, * each update will be returned as a separate entry, in the order of update @@ -98,7 +98,7 @@ public WBWIRocksIterator newIterator( /** * Create an iterator of the default column family. User can call - * {@link org.rocksdb.RocksIteratorInterface#seek(byte[])} to + * {@link org.forstdb.RocksIteratorInterface#seek(byte[])} to * search to the next entry of or after a key. Keys will be iterated in the * order given by index_comparator. For multiple updates on the same key, * each update will be returned as a separate entry, in the order of update @@ -112,7 +112,7 @@ public WBWIRocksIterator newIterator() { /** * Provides Read-Your-Own-Writes like functionality by - * creating a new Iterator that will use {@link org.rocksdb.WBWIRocksIterator} + * creating a new Iterator that will use {@link org.forstdb.WBWIRocksIterator} * as a delta and baseIterator as a base *

    * Updating write batch with the current key of the iterator is not safe. @@ -123,7 +123,7 @@ public WBWIRocksIterator newIterator() { * * @param columnFamilyHandle The column family to iterate over * @param baseIterator The base iterator, - * e.g. {@link org.rocksdb.RocksDB#newIterator()} + * e.g. {@link org.forstdb.RocksDB#newIterator()} * @return An iterator which shows a view comprised of both the database * point-in-time from baseIterator and modifications made in this write batch. */ @@ -135,7 +135,7 @@ public RocksIterator newIteratorWithBase( /** * Provides Read-Your-Own-Writes like functionality by - * creating a new Iterator that will use {@link org.rocksdb.WBWIRocksIterator} + * creating a new Iterator that will use {@link org.forstdb.WBWIRocksIterator} * as a delta and baseIterator as a base *

    * Updating write batch with the current key of the iterator is not safe. @@ -146,7 +146,7 @@ public RocksIterator newIteratorWithBase( * * @param columnFamilyHandle The column family to iterate over * @param baseIterator The base iterator, - * e.g. {@link org.rocksdb.RocksDB#newIterator()} + * e.g. {@link org.forstdb.RocksDB#newIterator()} * @param readOptions the read options, or null * @return An iterator which shows a view comprised of both the database * point-in-time from baseIterator and modifications made in this write batch. @@ -165,12 +165,12 @@ public RocksIterator newIteratorWithBase(final ColumnFamilyHandle columnFamilyHa /** * Provides Read-Your-Own-Writes like functionality by - * creating a new Iterator that will use {@link org.rocksdb.WBWIRocksIterator} + * creating a new Iterator that will use {@link org.forstdb.WBWIRocksIterator} * as a delta and baseIterator as a base. Operates on the default column * family. * * @param baseIterator The base iterator, - * e.g. {@link org.rocksdb.RocksDB#newIterator()} + * e.g. {@link org.forstdb.RocksDB#newIterator()} * @return An iterator which shows a view comprised of both the database * point-in-time from baseIterator and modifications made in this write batch. */ @@ -180,12 +180,12 @@ public RocksIterator newIteratorWithBase(final RocksIterator baseIterator) { /** * Provides Read-Your-Own-Writes like functionality by - * creating a new Iterator that will use {@link org.rocksdb.WBWIRocksIterator} + * creating a new Iterator that will use {@link org.forstdb.WBWIRocksIterator} * as a delta and baseIterator as a base. Operates on the default column * family. * * @param baseIterator The base iterator, - * e.g. {@link org.rocksdb.RocksDB#newIterator()} + * e.g. {@link org.forstdb.RocksDB#newIterator()} * @param readOptions the read options, or null * @return An iterator which shows a view comprised of both the database * point-in-time from baseIterator and modifications made in this write batch. diff --git a/java/src/main/java/org/rocksdb/WriteBufferManager.java b/java/src/main/java/org/forstdb/WriteBufferManager.java similarity index 83% rename from java/src/main/java/org/rocksdb/WriteBufferManager.java rename to java/src/main/java/org/forstdb/WriteBufferManager.java index 3364d6eab..18ecfea2b 100644 --- a/java/src/main/java/org/rocksdb/WriteBufferManager.java +++ b/java/src/main/java/org/forstdb/WriteBufferManager.java @@ -3,16 +3,12 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Java wrapper over native write_buffer_manager class */ public class WriteBufferManager extends RocksObject { - static { - RocksDB.loadLibrary(); - } - /** * Construct a new instance of WriteBufferManager. *

    @@ -28,7 +24,7 @@ public class WriteBufferManager extends RocksObject { */ public WriteBufferManager( final long bufferSizeBytes, final Cache cache, final boolean allowStall) { - super(newWriteBufferManager(bufferSizeBytes, cache.nativeHandle_, allowStall)); + super(newWriteBufferManagerInstance(bufferSizeBytes, cache.nativeHandle_, allowStall)); this.allowStall_ = allowStall; } @@ -40,6 +36,11 @@ public boolean allowStall() { return allowStall_; } + private static long newWriteBufferManagerInstance( + final long bufferSizeBytes, final long cacheHandle, final boolean allowStall) { + RocksDB.loadLibrary(); + return newWriteBufferManager(bufferSizeBytes, cacheHandle, allowStall); + } private static native long newWriteBufferManager( final long bufferSizeBytes, final long cacheHandle, final boolean allowStall); diff --git a/java/src/main/java/org/rocksdb/WriteOptions.java b/java/src/main/java/org/forstdb/WriteOptions.java similarity index 99% rename from java/src/main/java/org/rocksdb/WriteOptions.java rename to java/src/main/java/org/forstdb/WriteOptions.java index 7c184b094..26598d72e 100644 --- a/java/src/main/java/org/rocksdb/WriteOptions.java +++ b/java/src/main/java/org/forstdb/WriteOptions.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Options that control write operations. diff --git a/java/src/main/java/org/rocksdb/WriteStallCondition.java b/java/src/main/java/org/forstdb/WriteStallCondition.java similarity index 98% rename from java/src/main/java/org/rocksdb/WriteStallCondition.java rename to java/src/main/java/org/forstdb/WriteStallCondition.java index 98d9e2ce4..acc0dcdba 100644 --- a/java/src/main/java/org/rocksdb/WriteStallCondition.java +++ b/java/src/main/java/org/forstdb/WriteStallCondition.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public enum WriteStallCondition { DELAYED((byte) 0x0), diff --git a/java/src/main/java/org/rocksdb/WriteStallInfo.java b/java/src/main/java/org/forstdb/WriteStallInfo.java similarity index 99% rename from java/src/main/java/org/rocksdb/WriteStallInfo.java rename to java/src/main/java/org/forstdb/WriteStallInfo.java index 1cade0acb..dd48eb434 100644 --- a/java/src/main/java/org/rocksdb/WriteStallInfo.java +++ b/java/src/main/java/org/forstdb/WriteStallInfo.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Objects; diff --git a/java/src/main/java/org/forstdb/util/BufferUtil.java b/java/src/main/java/org/forstdb/util/BufferUtil.java new file mode 100644 index 000000000..8bec35922 --- /dev/null +++ b/java/src/main/java/org/forstdb/util/BufferUtil.java @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb.util; + +public class BufferUtil { + public static void CheckBounds(final int offset, final int len, final int size) { + if ((offset | len | (offset + len) | (size - (offset + len))) < 0) { + throw new IndexOutOfBoundsException( + String.format("offset(%d), len(%d), size(%d)", offset, len, size)); + } + } +} diff --git a/java/src/main/java/org/rocksdb/util/ByteUtil.java b/java/src/main/java/org/forstdb/util/ByteUtil.java similarity index 98% rename from java/src/main/java/org/rocksdb/util/ByteUtil.java rename to java/src/main/java/org/forstdb/util/ByteUtil.java index 5d64d5dcf..c42c4b690 100644 --- a/java/src/main/java/org/rocksdb/util/ByteUtil.java +++ b/java/src/main/java/org/forstdb/util/ByteUtil.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import java.nio.ByteBuffer; diff --git a/java/src/main/java/org/rocksdb/util/BytewiseComparator.java b/java/src/main/java/org/forstdb/util/BytewiseComparator.java similarity index 92% rename from java/src/main/java/org/rocksdb/util/BytewiseComparator.java rename to java/src/main/java/org/forstdb/util/BytewiseComparator.java index 9561b0a31..50c5fde9f 100644 --- a/java/src/main/java/org/rocksdb/util/BytewiseComparator.java +++ b/java/src/main/java/org/forstdb/util/BytewiseComparator.java @@ -3,13 +3,13 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; -import org.rocksdb.*; +import org.forstdb.*; import java.nio.ByteBuffer; -import static org.rocksdb.util.ByteUtil.memcmp; +import static org.forstdb.util.ByteUtil.memcmp; /** * This is a Java Native implementation of the C++ @@ -19,7 +19,7 @@ * less than their C++ counterparts due to the bridging overhead, * as such you likely don't want to use this apart from benchmarking * and you most likely instead wanted - * {@link org.rocksdb.BuiltinComparator#BYTEWISE_COMPARATOR} + * {@link org.forstdb.BuiltinComparator#BYTEWISE_COMPARATOR} */ public final class BytewiseComparator extends AbstractComparator { @@ -52,9 +52,9 @@ static int _compare(final ByteBuffer a, final ByteBuffer b) { return r; } + @SuppressWarnings("PMD.EmptyControlStatement") @Override - public void findShortestSeparator(final ByteBuffer start, - final ByteBuffer limit) { + public void findShortestSeparator(final ByteBuffer start, final ByteBuffer limit) { // Find length of common prefix final int minLength = Math.min(start.remaining(), limit.remaining()); int diffIndex = 0; diff --git a/java/src/main/java/org/rocksdb/util/Environment.java b/java/src/main/java/org/forstdb/util/Environment.java similarity index 91% rename from java/src/main/java/org/rocksdb/util/Environment.java rename to java/src/main/java/org/forstdb/util/Environment.java index 9ad51c7c7..57217a653 100644 --- a/java/src/main/java/org/rocksdb/util/Environment.java +++ b/java/src/main/java/org/forstdb/util/Environment.java @@ -1,14 +1,22 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb.util; +package org.forstdb.util; import java.io.File; import java.io.IOException; +import java.util.Locale; public class Environment { - private static String OS = System.getProperty("os.name").toLowerCase(); - private static String ARCH = System.getProperty("os.arch").toLowerCase(); + @SuppressWarnings("FieldMayBeFinal") + private static String OS = System.getProperty("os.name").toLowerCase(Locale.getDefault()); + @SuppressWarnings("FieldMayBeFinal") + private static String ARCH = System.getProperty("os.arch").toLowerCase(Locale.getDefault()); + @SuppressWarnings("FieldMayBeFinal") private static String MUSL_ENVIRONMENT = System.getenv("ROCKSDB_MUSL_LIBC"); + private static final String LIBC_MUSL_PREFIX = "libc.musl"; + + private static final String SPARCV9 = "sparcv9"; + /** * Will be lazily initialised by {@link #isMuslLibc()} instead of the previous static * initialisation. The lazy initialisation prevents Windows from reporting suspicious behaviour of @@ -28,6 +36,10 @@ public static boolean isS390x() { return ARCH.contains("s390x"); } + public static boolean isRiscv64() { + return ARCH.contains("riscv64"); + } + public static boolean isWindows() { return (OS.contains("win")); } @@ -70,6 +82,7 @@ public static boolean isMuslLibc() { * * @return true if the environment has a musl libc, false otherwise. */ + @SuppressWarnings("PMD.EmptyCatchBlock") static boolean initIsMuslLibc() { // consider explicit user setting from environment first if ("true".equalsIgnoreCase(MUSL_ENVIRONMENT)) { @@ -114,7 +127,7 @@ static boolean initIsMuslLibc() { return false; } for (final File f : libFiles) { - if (f.getName().startsWith("libc.musl")) { + if (f.getName().startsWith(LIBC_MUSL_PREFIX)) { return true; } } @@ -132,7 +145,7 @@ public static boolean isOpenBSD() { } public static boolean is64Bit() { - if (ARCH.indexOf("sparcv9") >= 0) { + if (ARCH.contains(SPARCV9)) { return true; } return (ARCH.indexOf("64") > 0); @@ -171,7 +184,7 @@ private static String getLibcPostfix() { public static String getJniLibraryName(final String name) { if (isUnix()) { final String arch = is64Bit() ? "64" : "32"; - if (isPowerPC() || isAarch64()) { + if (isPowerPC() || isAarch64() || isRiscv64()) { return String.format("%sjni-linux-%s%s", name, ARCH, getLibcPostfix()); } else if (isS390x()) { return String.format("%sjni-linux-%s", name, ARCH); diff --git a/java/src/main/java/org/rocksdb/util/IntComparator.java b/java/src/main/java/org/forstdb/util/IntComparator.java similarity index 90% rename from java/src/main/java/org/rocksdb/util/IntComparator.java rename to java/src/main/java/org/forstdb/util/IntComparator.java index cc096cd14..44dfa9f73 100644 --- a/java/src/main/java/org/rocksdb/util/IntComparator.java +++ b/java/src/main/java/org/forstdb/util/IntComparator.java @@ -3,10 +3,10 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; -import org.rocksdb.AbstractComparator; -import org.rocksdb.ComparatorOptions; +import org.forstdb.AbstractComparator; +import org.forstdb.ComparatorOptions; import java.nio.ByteBuffer; @@ -48,7 +48,7 @@ public int compare(final ByteBuffer a, final ByteBuffer b) { * * @return negative if a < b, 0 if a == b, positive otherwise */ - private final int compareIntKeys(final ByteBuffer a, final ByteBuffer b) { + private int compareIntKeys(final ByteBuffer a, final ByteBuffer b) { final int iA = a.getInt(); final int iB = b.getInt(); diff --git a/java/src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java b/java/src/main/java/org/forstdb/util/ReverseBytewiseComparator.java similarity index 89% rename from java/src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java rename to java/src/main/java/org/forstdb/util/ReverseBytewiseComparator.java index 4c06f80aa..87a8fb4e8 100644 --- a/java/src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java +++ b/java/src/main/java/org/forstdb/util/ReverseBytewiseComparator.java @@ -3,12 +3,12 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; -import org.rocksdb.AbstractComparator; -import org.rocksdb.BuiltinComparator; -import org.rocksdb.ComparatorOptions; -import org.rocksdb.Slice; +import org.forstdb.AbstractComparator; +import org.forstdb.BuiltinComparator; +import org.forstdb.ComparatorOptions; +import org.forstdb.Slice; import java.nio.ByteBuffer; @@ -38,9 +38,9 @@ public int compare(final ByteBuffer a, final ByteBuffer b) { return -BytewiseComparator._compare(a, b); } + @SuppressWarnings("PMD.EmptyControlStatement") @Override - public void findShortestSeparator(final ByteBuffer start, - final ByteBuffer limit) { + public void findShortestSeparator(final ByteBuffer start, final ByteBuffer limit) { // Find length of common prefix final int minLength = Math.min(start.remaining(), limit.remaining()); int diffIndex = 0; diff --git a/java/src/main/java/org/rocksdb/util/SizeUnit.java b/java/src/main/java/org/forstdb/util/SizeUnit.java similarity index 95% rename from java/src/main/java/org/rocksdb/util/SizeUnit.java rename to java/src/main/java/org/forstdb/util/SizeUnit.java index 0f717e8d4..313db8d44 100644 --- a/java/src/main/java/org/rocksdb/util/SizeUnit.java +++ b/java/src/main/java/org/forstdb/util/SizeUnit.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; public class SizeUnit { public static final long KB = 1024L; diff --git a/java/src/main/java/org/rocksdb/ExportImportFilesMetaData.java b/java/src/main/java/org/rocksdb/ExportImportFilesMetaData.java deleted file mode 100644 index 30892fc1e..000000000 --- a/java/src/main/java/org/rocksdb/ExportImportFilesMetaData.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -package org.rocksdb; - -import java.util.Arrays; -import java.util.List; - -/** - * The metadata that describes a column family. - */ -public class ExportImportFilesMetaData { - private final byte[] dbComparatorName; - private final LiveFileMetaData[] files; - - /** - * Called from JNI C++ - */ - public ExportImportFilesMetaData(final byte[] dbComparatorName, final LiveFileMetaData[] files) { - this.dbComparatorName = dbComparatorName; - this.files = files; - } - - /** - * The name of the db comparator. - * - * @return the dbComparatorName - */ - public byte[] dbComparatorName() { - return dbComparatorName; - } - - /** - * The metadata of all files in this column family. - * - * @return the levels files - */ - public List files() { - return Arrays.asList(files); - } - - public long newExportImportFilesMetaDataHandle() { - final long[] liveFileMetaDataHandles = new long[files.length]; - for (int i = 0; i < files.length; i++) { - liveFileMetaDataHandles[i] = files[i].newLiveFileMetaDataHandle(); - } - return newExportImportFilesMetaDataHandle( - dbComparatorName, dbComparatorName.length, liveFileMetaDataHandles); - } - - private native long newExportImportFilesMetaDataHandle(final byte[] dbComparatorName, - final int dbComparatorNameLen, final long[] liveFileMetaDataHandles); -} diff --git a/java/src/test/java/org/rocksdb/AbstractTransactionTest.java b/java/src/test/java/org/forstdb/AbstractTransactionTest.java similarity index 62% rename from java/src/test/java/org/rocksdb/AbstractTransactionTest.java rename to java/src/test/java/org/forstdb/AbstractTransactionTest.java index d57258009..09f69ba2c 100644 --- a/java/src/test/java/org/rocksdb/AbstractTransactionTest.java +++ b/java/src/test/java/org/forstdb/AbstractTransactionTest.java @@ -3,15 +3,17 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; +import java.util.function.Function; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -181,8 +183,10 @@ public void getPut_cf() throws RocksDBException { final ReadOptions readOptions = new ReadOptions(); final Transaction txn = dbContainer.beginTransaction()) { final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + assertThat(txn.get(readOptions, testCf, k1)).isNull(); assertThat(txn.get(testCf, readOptions, k1)).isNull(); txn.put(testCf, k1, v1); + assertThat(txn.get(readOptions, testCf, k1)).isEqualTo(v1); assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); } } @@ -200,6 +204,135 @@ public void getPut() throws RocksDBException { } } + @Test + public void getPutTargetBuffer_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final byte[] target = "overwrite1".getBytes(UTF_8); + GetStatus status = txn.get(readOptions, testCf, k1, target); + assertThat(status.status.getCode()).isEqualTo(Status.Code.NotFound); + assertThat(status.requiredSize).isEqualTo(0); + txn.put(testCf, k1, v1); + status = txn.get(readOptions, testCf, k1, target); + assertThat(status.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(status.requiredSize).isEqualTo(v1.length); + assertThat(target).isEqualTo("value1ite1".getBytes()); + } + } + + @Test + public void getPutTargetBuffer() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final byte[] target = "overwrite1".getBytes(UTF_8); + GetStatus status = txn.get(readOptions, k1, target); + assertThat(status.status.getCode()).isEqualTo(Status.Code.NotFound); + assertThat(status.requiredSize).isEqualTo(0); + txn.put(k1, v1); + status = txn.get(readOptions, k1, target); + assertThat(status.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(status.requiredSize).isEqualTo(v1.length); + assertThat(target).isEqualTo("value1ite1".getBytes()); + } + } + + public void getPutByteBuffer(final Function allocateBuffer) + throws RocksDBException { + final ByteBuffer k1 = allocateBuffer.apply(100).put("key1".getBytes(UTF_8)); + k1.flip(); + final ByteBuffer v1 = allocateBuffer.apply(100).put("value1".getBytes(UTF_8)); + v1.flip(); + final ByteBuffer vEmpty = allocateBuffer.apply(0); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ByteBuffer vGet = allocateBuffer.apply(100); + assertThat(txn.get(readOptions, k1, vGet).status.getCode()).isEqualTo(Status.Code.NotFound); + txn.put(k1, v1); + + final GetStatus getStatusError = txn.get(readOptions, k1, vEmpty); + assertThat(getStatusError.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(getStatusError.requiredSize).isEqualTo("value1".getBytes(UTF_8).length); + assertThat(vEmpty.position()).isEqualTo(0); + assertThat(vEmpty.remaining()).isEqualTo(0); + + vGet.put("12345".getBytes(UTF_8)); + + final GetStatus getStatus = txn.get(readOptions, k1, vGet); + assertThat(getStatusError.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(getStatusError.requiredSize).isEqualTo("value1".getBytes(UTF_8).length); + + vGet.put("67890".getBytes(UTF_8)); + vGet.flip(); + final byte[] bytes = new byte[vGet.limit()]; + vGet.get(bytes); + assertThat(new String(bytes, UTF_8)).isEqualTo("12345value167890"); + } + } + + @Test + public void getPutDirectByteBuffer() throws RocksDBException { + getPutByteBuffer(ByteBuffer::allocateDirect); + } + + @Test + public void getPutIndirectByteBuffer() throws RocksDBException { + getPutByteBuffer(ByteBuffer::allocate); + } + + public void getPutByteBuffer_cf(final Function allocateBuffer) + throws RocksDBException { + final ByteBuffer k1 = allocateBuffer.apply(100).put("key1".getBytes(UTF_8)); + k1.flip(); + final ByteBuffer v1 = allocateBuffer.apply(100).put("value1".getBytes(UTF_8)); + v1.flip(); + final ByteBuffer vEmpty = allocateBuffer.apply(0); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final ByteBuffer vGet = allocateBuffer.apply(100); + assertThat(txn.get(readOptions, testCf, k1, vGet).status.getCode()) + .isEqualTo(Status.Code.NotFound); + txn.put(testCf, k1, v1); + + final GetStatus getStatusError = txn.get(readOptions, testCf, k1, vEmpty); + assertThat(getStatusError.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(getStatusError.requiredSize).isEqualTo("value1".getBytes(UTF_8).length); + assertThat(vEmpty.position()).isEqualTo(0); + assertThat(vEmpty.remaining()).isEqualTo(0); + + vGet.put("12345".getBytes(UTF_8)); + final GetStatus getStatus = txn.get(readOptions, testCf, k1, vGet); + assertThat(getStatus.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(getStatus.requiredSize).isEqualTo("value1".getBytes(UTF_8).length); + vGet.put("67890".getBytes(UTF_8)); + vGet.flip(); + final byte[] bytes = new byte[vGet.limit()]; + vGet.get(bytes); + assertThat(new String(bytes, UTF_8)).isEqualTo("12345value167890"); + } + } + + @Test + public void getPutDirectByteBuffer_cf() throws RocksDBException { + getPutByteBuffer_cf(ByteBuffer::allocateDirect); + } + + @Test + public void getPutIndirectByteBuffer_cf() throws RocksDBException { + getPutByteBuffer_cf(ByteBuffer::allocate); + } + @Test public void multiGetPut_cf() throws RocksDBException { final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; @@ -300,6 +433,162 @@ public void getForUpdate() throws RocksDBException { } } + @Test + public void getForUpdateByteArray_cf_doValidate() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final byte[] vNonExistent = new byte[1]; + final GetStatus sNonExistent = + txn.getForUpdate(readOptions, testCf, k1, vNonExistent, true, true); + assertThat(sNonExistent.status.getCode()).isEqualTo(Status.Code.NotFound); + txn.put(testCf, k1, v1); + final byte[] vPartial = new byte[4]; + final GetStatus sPartial = txn.getForUpdate(readOptions, testCf, k1, vPartial, true, true); + assertThat(sPartial.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(sPartial.requiredSize).isEqualTo(v1.length); + assertThat(vPartial).isEqualTo(Arrays.copyOfRange(v1, 0, vPartial.length)); + + final byte[] vTotal = new byte[sPartial.requiredSize]; + final GetStatus sTotal = txn.getForUpdate(readOptions, testCf, k1, vTotal, true, true); + assertThat(sTotal.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(sTotal.requiredSize).isEqualTo(v1.length); + assertThat(vTotal).isEqualTo(v1); + } + } + + public void getForUpdateByteArray_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final byte[] vNonExistent = new byte[1]; + final GetStatus sNonExistent = txn.getForUpdate(readOptions, testCf, k1, vNonExistent, true); + assertThat(sNonExistent.status.getCode()).isEqualTo(Status.Code.NotFound); + txn.put(testCf, k1, v1); + final byte[] vPartial = new byte[4]; + final GetStatus sPartial = txn.getForUpdate(readOptions, testCf, k1, vPartial, true); + assertThat(sPartial.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(sPartial.requiredSize).isEqualTo(v1.length); + assertThat(vPartial).isEqualTo(Arrays.copyOfRange(v1, 0, vPartial.length)); + + final byte[] vTotal = new byte[sPartial.requiredSize]; + final GetStatus sTotal = txn.getForUpdate(readOptions, testCf, k1, vTotal, true); + assertThat(sTotal.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(sTotal.requiredSize).isEqualTo(v1.length); + assertThat(vTotal).isEqualTo(v1); + } + } + + @Test + public void getForUpdateByteArray() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final byte[] vNonExistent = new byte[1]; + final GetStatus sNonExistent = txn.getForUpdate(readOptions, k1, vNonExistent, true); + assertThat(sNonExistent.status.getCode()).isEqualTo(Status.Code.NotFound); + txn.put(k1, v1); + final byte[] vPartial = new byte[4]; + final GetStatus sPartial = txn.getForUpdate(readOptions, k1, vPartial, true); + assertThat(sPartial.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(sPartial.requiredSize).isEqualTo(v1.length); + assertThat(vPartial).isEqualTo(Arrays.copyOfRange(v1, 0, vPartial.length)); + + final byte[] vTotal = new byte[sPartial.requiredSize]; + final GetStatus sTotal = txn.getForUpdate(readOptions, k1, vTotal, true); + assertThat(sTotal.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(sTotal.requiredSize).isEqualTo(v1.length); + assertThat(vTotal).isEqualTo(v1); + } + } + + @Test + public void getForUpdateDirectByteBuffer() throws Exception { + getForUpdateByteBuffer(ByteBuffer::allocateDirect); + } + + @Test + public void getForUpdateIndirectByteBuffer() throws Exception { + getForUpdateByteBuffer(ByteBuffer::allocate); + } + + public void getForUpdateByteBuffer(final Function allocateBuffer) + throws Exception { + final ByteBuffer k1 = allocateBuffer.apply(20).put("key1".getBytes(UTF_8)); + k1.flip(); + final ByteBuffer v1 = allocateBuffer.apply(20).put("value1".getBytes(UTF_8)); + v1.flip(); + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ByteBuffer v1Read1 = allocateBuffer.apply(20); + final GetStatus getStatus1 = txn.getForUpdate(readOptions, k1, v1Read1, true); + assertThat(getStatus1.status.getCode()).isEqualTo(Status.Code.NotFound); + txn.put(k1, v1); + final ByteBuffer v1Read2 = allocateBuffer.apply(20); + final GetStatus getStatus2 = txn.getForUpdate(readOptions, k1, v1Read2, true); + assertThat(getStatus2.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(getStatus2.requiredSize).isEqualTo("value1".getBytes(UTF_8).length); + assertThat(v1Read2).isEqualTo(allocateBuffer.apply(20).put("value1".getBytes(UTF_8))); + } + } + + @Test + public void getForUpdateDirectByteBuffer_cf() throws Exception { + getForUpdateByteBuffer_cf(ByteBuffer::allocateDirect); + } + + @Test + public void getForUpdateIndirectByteBuffer_cf() throws Exception { + getForUpdateByteBuffer_cf(ByteBuffer::allocate); + } + + public void getForUpdateByteBuffer_cf(final Function allocateBuffer) + throws Exception { + final ByteBuffer k1 = allocateBuffer.apply(20).put("key1".getBytes(UTF_8)); + k1.flip(); + final ByteBuffer v1 = allocateBuffer.apply(20).put("value1".getBytes(UTF_8)); + v1.flip(); + final ByteBuffer k2 = allocateBuffer.apply(20).put("key2".getBytes(UTF_8)); + k2.flip(); + final ByteBuffer v2 = allocateBuffer.apply(20).put("value2".getBytes(UTF_8)); + v2.flip(); + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final ByteBuffer v1Read1 = allocateBuffer.apply(20); + final GetStatus getStatus1 = txn.getForUpdate(readOptions, testCf, k1, v1Read1, true); + assertThat(getStatus1.status.getCode()).isEqualTo(Status.Code.NotFound); + txn.put(k1, v1); + k1.flip(); + v1.flip(); + txn.put(testCf, k2, v2); + k2.flip(); + v2.flip(); + final ByteBuffer v1Read2 = allocateBuffer.apply(20); + final GetStatus getStatus2 = txn.getForUpdate(readOptions, testCf, k1, v1Read2, true); + assertThat(getStatus2.status.getCode()).isEqualTo(Status.Code.NotFound); + k1.flip(); + txn.put(testCf, k1, v1); + k1.flip(); + v1.flip(); + final ByteBuffer v1Read3 = allocateBuffer.apply(20); + final GetStatus getStatus3 = txn.getForUpdate(readOptions, testCf, k1, v1Read3, true); + assertThat(getStatus3.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(getStatus3.requiredSize).isEqualTo("value1".getBytes(UTF_8).length); + assertThat(v1Read3).isEqualTo(allocateBuffer.apply(20).put("value1".getBytes(UTF_8))); + } + } + @Test public void multiGetForUpdate_cf() throws RocksDBException { final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; @@ -401,6 +690,13 @@ public void getIterator() throws RocksDBException { assertThat(iterator.key()).isEqualTo(k1); assertThat(iterator.value()).isEqualTo(v1); } + + try (final RocksIterator iterator = txn.getIterator()) { + iterator.seek(k1); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo(k1); + assertThat(iterator.value()).isEqualTo(v1); + } } } @@ -422,6 +718,13 @@ public void getIterator_cf() throws RocksDBException { assertThat(iterator.key()).isEqualTo(k1); assertThat(iterator.value()).isEqualTo(v1); } + + try (final RocksIterator iterator = txn.getIterator(testCf)) { + iterator.seek(k1); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo(k1); + assertThat(iterator.value()).isEqualTo(v1); + } } } @@ -429,11 +732,15 @@ public void getIterator_cf() throws RocksDBException { public void merge_cf() throws RocksDBException { final byte[] k1 = "key1".getBytes(UTF_8); final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v2 = "value2".getBytes(UTF_8); try(final DBContainer dbContainer = startDb(); final Transaction txn = dbContainer.beginTransaction()) { final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); - txn.merge(testCf, k1, v1); + txn.put(testCf, k1, v1); + txn.merge(testCf, k1, v2); + assertThat(txn.get(new ReadOptions(), testCf, k1)).isEqualTo("value1**value2".getBytes()); + assertThat(txn.get(testCf, new ReadOptions(), k1)).isEqualTo("value1**value2".getBytes()); } } @@ -441,13 +748,94 @@ public void merge_cf() throws RocksDBException { public void merge() throws RocksDBException { final byte[] k1 = "key1".getBytes(UTF_8); final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v2 = "value2".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + txn.merge(k1, v2); + assertThat(txn.get(new ReadOptions(), k1)).isEqualTo("value1++value2".getBytes()); + } + } + + @Test + public void mergeDirectByteBuffer() throws RocksDBException { + final ByteBuffer k1 = ByteBuffer.allocateDirect(100).put("key1".getBytes(UTF_8)); + final ByteBuffer v1 = ByteBuffer.allocateDirect(100).put("value1".getBytes(UTF_8)); + final ByteBuffer v2 = ByteBuffer.allocateDirect(100).put("value2".getBytes(UTF_8)); + k1.flip(); + v1.flip(); + v2.flip(); + + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + k1.flip(); + v1.flip(); + txn.merge(k1, v2); + assertThat(txn.get(new ReadOptions(), "key1".getBytes(UTF_8))) + .isEqualTo("value1++value2".getBytes()); + } + } + + public void mergeIndirectByteBuffer() throws RocksDBException { + final ByteBuffer k1 = ByteBuffer.allocate(100).put("key1".getBytes(UTF_8)); + k1.flip(); + final ByteBuffer v1 = ByteBuffer.allocate(100).put("value1".getBytes(UTF_8)); + v1.flip(); + final ByteBuffer v2 = ByteBuffer.allocate(100).put("value2".getBytes(UTF_8)); + v2.flip(); try(final DBContainer dbContainer = startDb(); final Transaction txn = dbContainer.beginTransaction()) { - txn.merge(k1, v1); + txn.put(k1, v1); + txn.merge(k1, v2); + assertThat(txn.get(new ReadOptions(), "key1".getBytes(UTF_8))) + .isEqualTo("value1++value2".getBytes()); + } + } + + @Test + public void mergeDirectByteBuffer_cf() throws RocksDBException { + final ByteBuffer k1 = ByteBuffer.allocateDirect(100).put("key1".getBytes(UTF_8)); + final ByteBuffer v1 = ByteBuffer.allocateDirect(100).put("value1".getBytes(UTF_8)); + final ByteBuffer v2 = ByteBuffer.allocateDirect(100).put("value2".getBytes(UTF_8)); + k1.flip(); + v1.flip(); + v2.flip(); + + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.put(testCf, k1, v1); + k1.flip(); + v1.flip(); + txn.merge(testCf, k1, v2); + assertThat(txn.get(new ReadOptions(), testCf, "key1".getBytes(UTF_8))) + .isEqualTo("value1**value2".getBytes()); + assertThat(txn.get(testCf, new ReadOptions(), "key1".getBytes(UTF_8))) + .isEqualTo("value1**value2".getBytes()); } } + public void mergeIndirectByteBuffer_cf() throws RocksDBException { + final ByteBuffer k1 = ByteBuffer.allocate(100).put("key1".getBytes(UTF_8)); + k1.flip(); + final ByteBuffer v1 = ByteBuffer.allocate(100).put("value1".getBytes(UTF_8)); + v1.flip(); + final ByteBuffer v2 = ByteBuffer.allocate(100).put("value2".getBytes(UTF_8)); + v2.flip(); + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.put(testCf, k1, v1); + txn.merge(testCf, k1, v2); + assertThat(txn.get(new ReadOptions(), testCf, "key1".getBytes(UTF_8))) + .isEqualTo("value1**value2".getBytes()); + assertThat(txn.get(testCf, new ReadOptions(), "key1".getBytes(UTF_8))) + .isEqualTo("value1**value2".getBytes()); + } + } @Test public void delete_cf() throws RocksDBException { @@ -459,9 +847,11 @@ public void delete_cf() throws RocksDBException { final Transaction txn = dbContainer.beginTransaction()) { final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); txn.put(testCf, k1, v1); + assertThat(txn.get(readOptions, testCf, k1)).isEqualTo(v1); assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); txn.delete(testCf, k1); + assertThat(txn.get(readOptions, testCf, k1)).isNull(); assertThat(txn.get(testCf, readOptions, k1)).isNull(); } } @@ -495,11 +885,12 @@ public void delete_parts_cf() throws RocksDBException { final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); txn.put(testCf, keyParts, valueParts); assertThat(txn.get(testCf, readOptions, key)).isEqualTo(value); + assertThat(txn.get(readOptions, testCf, key)).isEqualTo(value); txn.delete(testCf, keyParts); - assertThat(txn.get(testCf, readOptions, key)) - .isNull(); + assertThat(txn.get(readOptions, testCf, key)).isNull(); + assertThat(txn.get(testCf, readOptions, key)).isNull(); } } @@ -532,8 +923,10 @@ public void getPutUntracked_cf() throws RocksDBException { final ReadOptions readOptions = new ReadOptions(); final Transaction txn = dbContainer.beginTransaction()) { final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + assertThat(txn.get(readOptions, testCf, k1)).isNull(); assertThat(txn.get(testCf, readOptions, k1)).isNull(); txn.putUntracked(testCf, k1, v1); + assertThat(txn.get(readOptions, testCf, k1)).isEqualTo(v1); assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); } } @@ -628,11 +1021,19 @@ public void multiGetPutAsListUntracked() throws RocksDBException { public void mergeUntracked_cf() throws RocksDBException { final byte[] k1 = "key1".getBytes(UTF_8); final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v2 = "value2".getBytes(UTF_8); try(final DBContainer dbContainer = startDb(); final Transaction txn = dbContainer.beginTransaction()) { final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); txn.mergeUntracked(testCf, k1, v1); + txn.mergeUntracked(testCf, k1, v2); + txn.commit(); + } + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + assertThat(txn.get(new ReadOptions(), testCf, k1)).isEqualTo("value1**value2".getBytes()); } } @@ -640,10 +1041,89 @@ public void mergeUntracked_cf() throws RocksDBException { public void mergeUntracked() throws RocksDBException { final byte[] k1 = "key1".getBytes(UTF_8); final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v2 = "value2".getBytes(UTF_8); try(final DBContainer dbContainer = startDb(); final Transaction txn = dbContainer.beginTransaction()) { txn.mergeUntracked(k1, v1); + txn.mergeUntracked(k1, v2); + txn.commit(); + } + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(new ReadOptions(), k1)).isEqualTo("value1++value2".getBytes()); + } + } + + @Test + public void mergeUntrackedByteBuffer() throws RocksDBException { + final ByteBuffer k1 = ByteBuffer.allocateDirect(20).put("key1".getBytes(UTF_8)); + final ByteBuffer v1 = ByteBuffer.allocateDirect(20).put("value1".getBytes(UTF_8)); + final ByteBuffer v2 = ByteBuffer.allocateDirect(20).put("value2".getBytes(UTF_8)); + k1.flip(); + v1.flip(); + v2.flip(); + + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.mergeUntracked(k1, v1); + k1.flip(); + v1.flip(); + txn.mergeUntracked(k1, v2); + k1.flip(); + v2.flip(); + txn.commit(); + } + + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ByteBuffer v = ByteBuffer.allocateDirect(20); + final GetStatus status = txn.get(new ReadOptions(), k1, v); + assertThat(status.status.getCode()).isEqualTo(Status.Code.Ok); + k1.flip(); + v.flip(); + final int expectedLength = "value1++value2".length(); + assertThat(v.remaining()).isEqualTo(expectedLength); + final byte[] vBytes = new byte[expectedLength]; + v.get(vBytes); + assertThat(vBytes).isEqualTo("value1++value2".getBytes()); + } + } + + @Test + public void mergeUntrackedByteBuffer_cf() throws RocksDBException { + final ByteBuffer k1 = ByteBuffer.allocateDirect(20).put("key1".getBytes(UTF_8)); + final ByteBuffer v1 = ByteBuffer.allocateDirect(20).put("value1".getBytes(UTF_8)); + final ByteBuffer v2 = ByteBuffer.allocateDirect(20).put("value2".getBytes(UTF_8)); + k1.flip(); + v1.flip(); + v2.flip(); + + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.mergeUntracked(testCf, k1, v1); + k1.flip(); + v1.flip(); + txn.mergeUntracked(testCf, k1, v2); + k1.flip(); + v2.flip(); + txn.commit(); + } + + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ByteBuffer v = ByteBuffer.allocateDirect(20); + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final GetStatus status = txn.get(new ReadOptions(), testCf, k1, v); + assertThat(status.status.getCode()).isEqualTo(Status.Code.Ok); + k1.flip(); + v.flip(); + final int expectedLength = "value1++value2".length(); + assertThat(v.remaining()).isEqualTo(expectedLength); + final byte[] vBytes = new byte[expectedLength]; + v.get(vBytes); + assertThat(vBytes).isEqualTo("value1**value2".getBytes()); } } @@ -657,9 +1137,11 @@ public void deleteUntracked_cf() throws RocksDBException { final Transaction txn = dbContainer.beginTransaction()) { final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); txn.put(testCf, k1, v1); + assertThat(txn.get(readOptions, testCf, k1)).isEqualTo(v1); assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); txn.deleteUntracked(testCf, k1); + assertThat(txn.get(readOptions, testCf, k1)).isNull(); assertThat(txn.get(testCf, readOptions, k1)).isNull(); } } @@ -692,9 +1174,11 @@ public void deleteUntracked_parts_cf() throws RocksDBException { final Transaction txn = dbContainer.beginTransaction()) { final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); txn.put(testCf, keyParts, valueParts); + assertThat(txn.get(readOptions, testCf, key)).isEqualTo(value); assertThat(txn.get(testCf, readOptions, key)).isEqualTo(value); txn.deleteUntracked(testCf, keyParts); + assertThat(txn.get(readOptions, testCf, key)).isNull(); assertThat(txn.get(testCf, readOptions, key)).isNull(); } } diff --git a/java/src/test/java/org/rocksdb/BackupEngineOptionsTest.java b/java/src/test/java/org/forstdb/BackupEngineOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/BackupEngineOptionsTest.java rename to java/src/test/java/org/forstdb/BackupEngineOptionsTest.java index b07f8d33c..e9f6087f3 100644 --- a/java/src/test/java/org/rocksdb/BackupEngineOptionsTest.java +++ b/java/src/test/java/org/forstdb/BackupEngineOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/BackupEngineTest.java b/java/src/test/java/org/forstdb/BackupEngineTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/BackupEngineTest.java rename to java/src/test/java/org/forstdb/BackupEngineTest.java index 67145f846..9b136a527 100644 --- a/java/src/test/java/org/rocksdb/BackupEngineTest.java +++ b/java/src/test/java/org/forstdb/BackupEngineTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/rocksdb/BlobOptionsTest.java b/java/src/test/java/org/forstdb/BlobOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/BlobOptionsTest.java rename to java/src/test/java/org/forstdb/BlobOptionsTest.java index a0a2af84a..a7895fe34 100644 --- a/java/src/test/java/org/rocksdb/BlobOptionsTest.java +++ b/java/src/test/java/org/forstdb/BlobOptionsTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java b/java/src/test/java/org/forstdb/BlockBasedTableConfigTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java rename to java/src/test/java/org/forstdb/BlockBasedTableConfigTest.java index 13247d1e6..51b23ede8 100644 --- a/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java +++ b/java/src/test/java/org/forstdb/BlockBasedTableConfigTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; diff --git a/java/src/test/java/org/rocksdb/BuiltinComparatorTest.java b/java/src/test/java/org/forstdb/BuiltinComparatorTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/BuiltinComparatorTest.java rename to java/src/test/java/org/forstdb/BuiltinComparatorTest.java index e238ae07b..70e7ccf17 100644 --- a/java/src/test/java/org/rocksdb/BuiltinComparatorTest.java +++ b/java/src/test/java/org/forstdb/BuiltinComparatorTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/rocksdb/ByteBufferUnsupportedOperationTest.java b/java/src/test/java/org/forstdb/ByteBufferUnsupportedOperationTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/ByteBufferUnsupportedOperationTest.java rename to java/src/test/java/org/forstdb/ByteBufferUnsupportedOperationTest.java index f596f573f..b8d38cbd2 100644 --- a/java/src/test/java/org/rocksdb/ByteBufferUnsupportedOperationTest.java +++ b/java/src/test/java/org/forstdb/ByteBufferUnsupportedOperationTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.nio.charset.StandardCharsets; import java.util.*; @@ -12,7 +12,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.ReverseBytewiseComparator; +import org.forstdb.util.ReverseBytewiseComparator; public class ByteBufferUnsupportedOperationTest { @ClassRule diff --git a/java/src/test/java/org/rocksdb/BytewiseComparatorRegressionTest.java b/java/src/test/java/org/forstdb/BytewiseComparatorRegressionTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/BytewiseComparatorRegressionTest.java rename to java/src/test/java/org/forstdb/BytewiseComparatorRegressionTest.java index 13aa6c2bd..84cfeace5 100644 --- a/java/src/test/java/org/rocksdb/BytewiseComparatorRegressionTest.java +++ b/java/src/test/java/org/forstdb/BytewiseComparatorRegressionTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.junit.Assert.assertArrayEquals; @@ -14,7 +14,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.BytewiseComparator; +import org.forstdb.util.BytewiseComparator; /** * This test confirms that the following issues were in fact resolved diff --git a/java/src/test/java/org/rocksdb/CheckPointTest.java b/java/src/test/java/org/forstdb/CheckPointTest.java similarity index 90% rename from java/src/test/java/org/rocksdb/CheckPointTest.java rename to java/src/test/java/org/forstdb/CheckPointTest.java index 744d96f5c..eb846e492 100644 --- a/java/src/test/java/org/rocksdb/CheckPointTest.java +++ b/java/src/test/java/org/forstdb/CheckPointTest.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; @@ -66,16 +66,10 @@ public void exportColumnFamily() throws RocksDBException { ExportImportFilesMetaData metadata1 = checkpoint.exportColumnFamily(db.getDefaultColumnFamily(), checkpointFolder.getRoot().getAbsolutePath() + "/export_column_family1"); - assertThat(metadata1.files().size()).isEqualTo(1); - assertThat(metadata1.dbComparatorName()) - .isEqualTo("leveldb.BytewiseComparator".getBytes()); db.put("key2".getBytes(), "value2".getBytes()); ExportImportFilesMetaData metadata2 = checkpoint.exportColumnFamily(db.getDefaultColumnFamily(), checkpointFolder.getRoot().getAbsolutePath() + "/export_column_family2"); - assertThat(metadata2.files().size()).isEqualTo(2); - assertThat(metadata2.dbComparatorName()) - .isEqualTo("leveldb.BytewiseComparator".getBytes()); } } } diff --git a/java/src/test/java/org/rocksdb/ClockCacheTest.java b/java/src/test/java/org/forstdb/ClockCacheTest.java similarity index 96% rename from java/src/test/java/org/rocksdb/ClockCacheTest.java rename to java/src/test/java/org/forstdb/ClockCacheTest.java index 718c24f70..8e466e1f4 100644 --- a/java/src/test/java/org/rocksdb/ClockCacheTest.java +++ b/java/src/test/java/org/forstdb/ClockCacheTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java b/java/src/test/java/org/forstdb/ColumnFamilyOptionsTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java rename to java/src/test/java/org/forstdb/ColumnFamilyOptionsTest.java index a5fe8cef7..aac9e0f24 100644 --- a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java +++ b/java/src/test/java/org/forstdb/ColumnFamilyOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; @@ -13,7 +13,7 @@ import java.util.*; import org.junit.ClassRule; import org.junit.Test; -import org.rocksdb.test.RemoveEmptyValueCompactionFilterFactory; +import org.forstdb.test.RemoveEmptyValueCompactionFilterFactory; public class ColumnFamilyOptionsTest { @@ -709,4 +709,14 @@ public void cfPaths() throws IOException { assertThat(options.cfPaths()).isEqualTo(paths); } } + + @Test + public void memtableMaxRangeDeletions() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + assertThat(options.memtableMaxRangeDeletions()).isEqualTo(0); + final int val = 32; + assertThat(options.setMemtableMaxRangeDeletions(val)).isEqualTo(options); + assertThat(options.memtableMaxRangeDeletions()).isEqualTo(val); + } + } } diff --git a/java/src/test/java/org/rocksdb/ColumnFamilyTest.java b/java/src/test/java/org/forstdb/ColumnFamilyTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/ColumnFamilyTest.java rename to java/src/test/java/org/forstdb/ColumnFamilyTest.java index fb8a45085..a629ccc95 100644 --- a/java/src/test/java/org/rocksdb/ColumnFamilyTest.java +++ b/java/src/test/java/org/forstdb/ColumnFamilyTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/CompactRangeOptionsTest.java b/java/src/test/java/org/forstdb/CompactRangeOptionsTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/CompactRangeOptionsTest.java rename to java/src/test/java/org/forstdb/CompactRangeOptionsTest.java index 549b74beb..1b7941365 100644 --- a/java/src/test/java/org/rocksdb/CompactRangeOptionsTest.java +++ b/java/src/test/java/org/forstdb/CompactRangeOptionsTest.java @@ -3,10 +3,10 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; -import org.rocksdb.CompactRangeOptions.BottommostLevelCompaction; +import org.forstdb.CompactRangeOptions.BottommostLevelCompaction; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/CompactionFilterFactoryTest.java b/java/src/test/java/org/forstdb/CompactionFilterFactoryTest.java similarity index 96% rename from java/src/test/java/org/rocksdb/CompactionFilterFactoryTest.java rename to java/src/test/java/org/forstdb/CompactionFilterFactoryTest.java index 35a14eb54..7209161f5 100644 --- a/java/src/test/java/org/rocksdb/CompactionFilterFactoryTest.java +++ b/java/src/test/java/org/forstdb/CompactionFilterFactoryTest.java @@ -3,12 +3,12 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.test.RemoveEmptyValueCompactionFilterFactory; +import org.forstdb.test.RemoveEmptyValueCompactionFilterFactory; import java.util.ArrayList; import java.util.Arrays; diff --git a/java/src/test/java/org/rocksdb/CompactionJobInfoTest.java b/java/src/test/java/org/forstdb/CompactionJobInfoTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/CompactionJobInfoTest.java rename to java/src/test/java/org/forstdb/CompactionJobInfoTest.java index c71b0da16..6c38c0c01 100644 --- a/java/src/test/java/org/rocksdb/CompactionJobInfoTest.java +++ b/java/src/test/java/org/forstdb/CompactionJobInfoTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/CompactionJobStatsTest.java b/java/src/test/java/org/forstdb/CompactionJobStatsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/CompactionJobStatsTest.java rename to java/src/test/java/org/forstdb/CompactionJobStatsTest.java index 5c1eb2aab..30df1f4df 100644 --- a/java/src/test/java/org/rocksdb/CompactionJobStatsTest.java +++ b/java/src/test/java/org/forstdb/CompactionJobStatsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/CompactionOptionsFIFOTest.java b/java/src/test/java/org/forstdb/CompactionOptionsFIFOTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/CompactionOptionsFIFOTest.java rename to java/src/test/java/org/forstdb/CompactionOptionsFIFOTest.java index 841615e67..6da11aa52 100644 --- a/java/src/test/java/org/rocksdb/CompactionOptionsFIFOTest.java +++ b/java/src/test/java/org/forstdb/CompactionOptionsFIFOTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/CompactionOptionsTest.java b/java/src/test/java/org/forstdb/CompactionOptionsTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/CompactionOptionsTest.java rename to java/src/test/java/org/forstdb/CompactionOptionsTest.java index 9b7d79694..6f070c4d3 100644 --- a/java/src/test/java/org/rocksdb/CompactionOptionsTest.java +++ b/java/src/test/java/org/forstdb/CompactionOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/CompactionOptionsUniversalTest.java b/java/src/test/java/org/forstdb/CompactionOptionsUniversalTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/CompactionOptionsUniversalTest.java rename to java/src/test/java/org/forstdb/CompactionOptionsUniversalTest.java index 5e2d195b6..6aa0ef2cc 100644 --- a/java/src/test/java/org/rocksdb/CompactionOptionsUniversalTest.java +++ b/java/src/test/java/org/forstdb/CompactionOptionsUniversalTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/CompactionPriorityTest.java b/java/src/test/java/org/forstdb/CompactionPriorityTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/CompactionPriorityTest.java rename to java/src/test/java/org/forstdb/CompactionPriorityTest.java index b078e132f..9902b9c97 100644 --- a/java/src/test/java/org/rocksdb/CompactionPriorityTest.java +++ b/java/src/test/java/org/forstdb/CompactionPriorityTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/CompactionStopStyleTest.java b/java/src/test/java/org/forstdb/CompactionStopStyleTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/CompactionStopStyleTest.java rename to java/src/test/java/org/forstdb/CompactionStopStyleTest.java index 4c8a20950..978aee632 100644 --- a/java/src/test/java/org/rocksdb/CompactionStopStyleTest.java +++ b/java/src/test/java/org/forstdb/CompactionStopStyleTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/ComparatorOptionsTest.java b/java/src/test/java/org/forstdb/ComparatorOptionsTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/ComparatorOptionsTest.java rename to java/src/test/java/org/forstdb/ComparatorOptionsTest.java index 3e90b9f10..ea6486378 100644 --- a/java/src/test/java/org/rocksdb/ComparatorOptionsTest.java +++ b/java/src/test/java/org/forstdb/ComparatorOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/CompressionOptionsTest.java b/java/src/test/java/org/forstdb/CompressionOptionsTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/CompressionOptionsTest.java rename to java/src/test/java/org/forstdb/CompressionOptionsTest.java index 116552c32..c11c57af9 100644 --- a/java/src/test/java/org/rocksdb/CompressionOptionsTest.java +++ b/java/src/test/java/org/forstdb/CompressionOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/CompressionTypesTest.java b/java/src/test/java/org/forstdb/CompressionTypesTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/CompressionTypesTest.java rename to java/src/test/java/org/forstdb/CompressionTypesTest.java index a983f471a..761d4eec0 100644 --- a/java/src/test/java/org/rocksdb/CompressionTypesTest.java +++ b/java/src/test/java/org/forstdb/CompressionTypesTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/ConcurrentTaskLimiterTest.java b/java/src/test/java/org/forstdb/ConcurrentTaskLimiterTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/ConcurrentTaskLimiterTest.java rename to java/src/test/java/org/forstdb/ConcurrentTaskLimiterTest.java index 165f4f24c..8e6b5b02c 100644 --- a/java/src/test/java/org/rocksdb/ConcurrentTaskLimiterTest.java +++ b/java/src/test/java/org/forstdb/ConcurrentTaskLimiterTest.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.junit.Assert.assertEquals; diff --git a/java/src/test/java/org/rocksdb/DBOptionsTest.java b/java/src/test/java/org/forstdb/DBOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/DBOptionsTest.java rename to java/src/test/java/org/forstdb/DBOptionsTest.java index 882015f3e..ffa22a231 100644 --- a/java/src/test/java/org/rocksdb/DBOptionsTest.java +++ b/java/src/test/java/org/forstdb/DBOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; @@ -453,6 +453,7 @@ public void setWriteBufferManagerWithZeroBufferSize() throws RocksDBException { } } + @SuppressWarnings("deprecated") @Test public void accessHintOnCompactionStart() { try(final DBOptions opt = new DBOptions()) { @@ -885,6 +886,8 @@ public void onMemTableSealed(final MemTableInfo memTableInfo) { wasCalled2.set(true); } }) { + assertThat(options.setListeners(null)).isEqualTo(options); + assertThat(options.listeners().size()).isEqualTo(0); assertThat(options.setListeners(Arrays.asList(el1, el2))).isEqualTo(options); final List listeners = options.listeners(); assertEquals(el1, listeners.get(0)); diff --git a/java/src/test/java/org/rocksdb/DefaultEnvTest.java b/java/src/test/java/org/forstdb/DefaultEnvTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/DefaultEnvTest.java rename to java/src/test/java/org/forstdb/DefaultEnvTest.java index 3fb563ecb..a53d1de0f 100644 --- a/java/src/test/java/org/rocksdb/DefaultEnvTest.java +++ b/java/src/test/java/org/forstdb/DefaultEnvTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/rocksdb/DirectSliceTest.java b/java/src/test/java/org/forstdb/DirectSliceTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/DirectSliceTest.java rename to java/src/test/java/org/forstdb/DirectSliceTest.java index 67385345c..c14f4925b 100644 --- a/java/src/test/java/org/rocksdb/DirectSliceTest.java +++ b/java/src/test/java/org/forstdb/DirectSliceTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/EnvOptionsTest.java b/java/src/test/java/org/forstdb/EnvOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/EnvOptionsTest.java rename to java/src/test/java/org/forstdb/EnvOptionsTest.java index 0f3d8e234..fcf1bcf87 100644 --- a/java/src/test/java/org/rocksdb/EnvOptionsTest.java +++ b/java/src/test/java/org/forstdb/EnvOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/EventListenerTest.java b/java/src/test/java/org/forstdb/EventListenerTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/EventListenerTest.java rename to java/src/test/java/org/forstdb/EventListenerTest.java index 84be232f9..4d5729586 100644 --- a/java/src/test/java/org/rocksdb/EventListenerTest.java +++ b/java/src/test/java/org/forstdb/EventListenerTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; @@ -17,8 +17,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.AbstractEventListener.EnabledEventCallback; -import org.rocksdb.test.TestableEventListener; +import org.forstdb.AbstractEventListener.EnabledEventCallback; +import org.forstdb.test.TestableEventListener; public class EventListenerTest { @ClassRule diff --git a/java/src/test/java/org/rocksdb/FilterTest.java b/java/src/test/java/org/forstdb/FilterTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/FilterTest.java rename to java/src/test/java/org/forstdb/FilterTest.java index e308ffefb..d2e93e4e2 100644 --- a/java/src/test/java/org/rocksdb/FilterTest.java +++ b/java/src/test/java/org/forstdb/FilterTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/FlinkCompactionFilterTest.java b/java/src/test/java/org/forstdb/FlinkCompactionFilterTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/FlinkCompactionFilterTest.java rename to java/src/test/java/org/forstdb/FlinkCompactionFilterTest.java index 40320e9d5..87bbb6bbe 100644 --- a/java/src/test/java/org/rocksdb/FlinkCompactionFilterTest.java +++ b/java/src/test/java/org/forstdb/FlinkCompactionFilterTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; @@ -31,8 +31,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.FlinkCompactionFilter.StateType; -import org.rocksdb.FlinkCompactionFilter.TimeProvider; +import org.forstdb.FlinkCompactionFilter.StateType; +import org.forstdb.FlinkCompactionFilter.TimeProvider; public class FlinkCompactionFilterTest { private static final int LONG_LENGTH = 8; diff --git a/java/src/test/java/org/rocksdb/FlushOptionsTest.java b/java/src/test/java/org/forstdb/FlushOptionsTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/FlushOptionsTest.java rename to java/src/test/java/org/forstdb/FlushOptionsTest.java index f90ae911d..2c0e268b0 100644 --- a/java/src/test/java/org/rocksdb/FlushOptionsTest.java +++ b/java/src/test/java/org/forstdb/FlushOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/FlushTest.java b/java/src/test/java/org/forstdb/FlushTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/FlushTest.java rename to java/src/test/java/org/forstdb/FlushTest.java index 1a354f4ce..358091561 100644 --- a/java/src/test/java/org/rocksdb/FlushTest.java +++ b/java/src/test/java/org/forstdb/FlushTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/forstdb/HyperClockCacheTest.java b/java/src/test/java/org/forstdb/HyperClockCacheTest.java new file mode 100644 index 000000000..5bbc93db8 --- /dev/null +++ b/java/src/test/java/org/forstdb/HyperClockCacheTest.java @@ -0,0 +1,36 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class HyperClockCacheTest { + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void newHyperClockCache() throws RocksDBException { + RocksDB.loadLibrary(); + try (Cache cache = new HyperClockCache(1024 * 1024, 0, 8, false)) { + BlockBasedTableConfig tableConfing = new BlockBasedTableConfig(); + tableConfing.setBlockCache(cache); + try (Options options = new Options()) { + options.setTableFormatConfig(tableConfing); + options.setCreateIfMissing(true); + try (RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put("testKey".getBytes(), "testData".getBytes()); + // no op + assertThat(cache.getUsage()).isGreaterThanOrEqualTo(0); + assertThat(cache.getPinnedUsage()).isGreaterThanOrEqualTo(0); + } + } + } + } +} diff --git a/java/src/test/java/org/rocksdb/ImportColumnFamilyTest.java b/java/src/test/java/org/forstdb/ImportColumnFamilyTest.java similarity index 93% rename from java/src/test/java/org/rocksdb/ImportColumnFamilyTest.java rename to java/src/test/java/org/forstdb/ImportColumnFamilyTest.java index 476900601..040f2a14d 100644 --- a/java/src/test/java/org/rocksdb/ImportColumnFamilyTest.java +++ b/java/src/test/java/org/forstdb/ImportColumnFamilyTest.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -19,7 +19,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.BytewiseComparator; +import org.forstdb.util.BytewiseComparator; public class ImportColumnFamilyTest { private static final String SST_FILE_NAME = "test.sst"; @@ -40,13 +40,14 @@ public void testImportColumnFamily() throws RocksDBException { db.put("key".getBytes(), "value".getBytes()); db.put("key1".getBytes(), "value1".getBytes()); - try (final Checkpoint checkpoint = Checkpoint.create(db)) { + try (final Checkpoint checkpoint = Checkpoint.create(db); + final ImportColumnFamilyOptions importColumnFamilyOptions = + new ImportColumnFamilyOptions()) { ExportImportFilesMetaData default_cf_metadata = checkpoint.exportColumnFamily(db.getDefaultColumnFamily(), checkpointFolder.getRoot().getAbsolutePath() + "/default_cf_metadata"); ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor("new_cf".getBytes()); - ImportColumnFamilyOptions importColumnFamilyOptions = new ImportColumnFamilyOptions(); final ColumnFamilyHandle importCfHandle = db.createColumnFamilyWithImport( columnFamilyDescriptor, importColumnFamilyOptions, default_cf_metadata); assertThat(db.get(importCfHandle, "key".getBytes())).isEqualTo("value".getBytes()); @@ -67,7 +68,9 @@ public void ImportMultiColumnFamilyTest() throws RocksDBException { db2.put("key2".getBytes(), "value2".getBytes()); db2.put("key3".getBytes(), "value3".getBytes()); try (final Checkpoint checkpoint1 = Checkpoint.create(db1); - final Checkpoint checkpoint2 = Checkpoint.create(db2);) { + final Checkpoint checkpoint2 = Checkpoint.create(db2); + final ImportColumnFamilyOptions importColumnFamilyOptions = + new ImportColumnFamilyOptions()) { ExportImportFilesMetaData default_cf_metadata1 = checkpoint1.exportColumnFamily(db1.getDefaultColumnFamily(), checkpointFolder.getRoot().getAbsolutePath() + "/default_cf_metadata1"); @@ -77,7 +80,6 @@ public void ImportMultiColumnFamilyTest() throws RocksDBException { ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor("new_cf".getBytes()); - ImportColumnFamilyOptions importColumnFamilyOptions = new ImportColumnFamilyOptions(); List importMetaDatas = new ArrayList(); importMetaDatas.add(default_cf_metadata1); diff --git a/java/src/test/java/org/rocksdb/InfoLogLevelTest.java b/java/src/test/java/org/forstdb/InfoLogLevelTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/InfoLogLevelTest.java rename to java/src/test/java/org/forstdb/InfoLogLevelTest.java index 90b0b4e2d..6afd1f783 100644 --- a/java/src/test/java/org/rocksdb/InfoLogLevelTest.java +++ b/java/src/test/java/org/forstdb/InfoLogLevelTest.java @@ -1,11 +1,11 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.Environment; +import org.forstdb.util.Environment; import java.io.IOException; diff --git a/java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java b/java/src/test/java/org/forstdb/IngestExternalFileOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java rename to java/src/test/java/org/forstdb/IngestExternalFileOptionsTest.java index 230694615..535156580 100644 --- a/java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java +++ b/java/src/test/java/org/forstdb/IngestExternalFileOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/forstdb/KeyExistsTest.java b/java/src/test/java/org/forstdb/KeyExistsTest.java new file mode 100644 index 000000000..150411cb7 --- /dev/null +++ b/java/src/test/java/org/forstdb/KeyExistsTest.java @@ -0,0 +1,229 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.forstdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.*; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +public class KeyExistsTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule public ExpectedException exceptionRule = ExpectedException.none(); + + List cfDescriptors; + List columnFamilyHandleList = new ArrayList<>(); + RocksDB db; + @Before + public void before() throws RocksDBException { + cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + + db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList); + } + + @After + public void after() { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + db.close(); + } + + @Test + public void keyExists() throws RocksDBException { + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + boolean exists = db.keyExists("key".getBytes(UTF_8)); + assertThat(exists).isTrue(); + exists = db.keyExists("key2".getBytes(UTF_8)); + assertThat(exists).isFalse(); + } + + @Test + public void keyExistsColumnFamily() throws RocksDBException { + byte[] key1 = "keyBBCF0".getBytes(UTF_8); + byte[] key2 = "keyBBCF1".getBytes(UTF_8); + db.put(columnFamilyHandleList.get(0), key1, "valueBBCF0".getBytes(UTF_8)); + db.put(columnFamilyHandleList.get(1), key2, "valueBBCF1".getBytes(UTF_8)); + + assertThat(db.keyExists(columnFamilyHandleList.get(0), key1)).isTrue(); + assertThat(db.keyExists(columnFamilyHandleList.get(0), key2)).isFalse(); + + assertThat(db.keyExists(columnFamilyHandleList.get(1), key1)).isFalse(); + assertThat(db.keyExists(columnFamilyHandleList.get(1), key2)).isTrue(); + } + + @Test + public void keyExistsColumnFamilyReadOptions() throws RocksDBException { + try (final ReadOptions readOptions = new ReadOptions()) { + byte[] key1 = "keyBBCF0".getBytes(UTF_8); + byte[] key2 = "keyBBCF1".getBytes(UTF_8); + db.put(columnFamilyHandleList.get(0), key1, "valueBBCF0".getBytes(UTF_8)); + db.put(columnFamilyHandleList.get(1), key2, "valueBBCF1".getBytes(UTF_8)); + + assertThat(db.keyExists(columnFamilyHandleList.get(0), readOptions, key1)).isTrue(); + assertThat(db.keyExists(columnFamilyHandleList.get(0), readOptions, key2)).isFalse(); + + assertThat(db.keyExists(columnFamilyHandleList.get(1), readOptions, key1)).isFalse(); + assertThat(db.keyExists(columnFamilyHandleList.get(1), readOptions, key2)).isTrue(); + } + } + + @Test + public void keyExistsReadOptions() throws RocksDBException { + try (final ReadOptions readOptions = new ReadOptions()) { + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + boolean exists = db.keyExists(readOptions, "key".getBytes(UTF_8)); + assertThat(exists).isTrue(); + exists = db.keyExists("key2".getBytes(UTF_8)); + assertThat(exists).isFalse(); + } + } + + @Test + public void keyExistsAfterDelete() throws RocksDBException { + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + boolean exists = db.keyExists(null, null, "key".getBytes(UTF_8), 0, 3); + assertThat(exists).isTrue(); + db.delete("key".getBytes(UTF_8)); + exists = db.keyExists(null, null, "key".getBytes(UTF_8), 0, 3); + assertThat(exists).isFalse(); + } + + @Test + public void keyExistsArrayIndexOutOfBoundsException() throws RocksDBException { + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + exceptionRule.expect(IndexOutOfBoundsException.class); + db.keyExists(null, null, "key".getBytes(UTF_8), 0, 5); + } + + @Test() + public void keyExistsArrayIndexOutOfBoundsExceptionWrongOffset() throws RocksDBException { + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + exceptionRule.expect(IndexOutOfBoundsException.class); + db.keyExists(null, null, "key".getBytes(UTF_8), 6, 2); + } + + @Test + public void keyExistsDirectByteBuffer() throws RocksDBException { + byte[] key = "key".getBytes(UTF_8); + + db.put(key, "value".getBytes(UTF_8)); + ByteBuffer buff = ByteBuffer.allocateDirect(key.length); + buff.put(key); + buff.flip(); + boolean exists = db.keyExists(buff); + assertThat(exists).isTrue(); + } + + @Test + public void keyExistsDirectByteBufferReadOptions() throws RocksDBException { + try (final ReadOptions readOptions = new ReadOptions()) { + byte[] key = "key".getBytes(UTF_8); + + db.put(key, "value".getBytes(UTF_8)); + ByteBuffer buff = ByteBuffer.allocateDirect(key.length); + buff.put(key); + buff.flip(); + + boolean exists = db.keyExists(buff); + assertThat(exists).isTrue(); + } + } + + @Test + public void keyExistsDirectByteBufferAfterDelete() throws RocksDBException { + byte[] key = "key".getBytes(UTF_8); + + db.put(key, "value".getBytes(UTF_8)); + ByteBuffer buff = ByteBuffer.allocateDirect(key.length); + buff.put(key); + buff.flip(); + boolean exists = db.keyExists(buff); + assertThat(exists).isTrue(); + db.delete(key); + exists = db.keyExists(buff); + assertThat(exists).isFalse(); + } + + @Test + public void keyExistsDirectByteBufferColumnFamily() throws RocksDBException { + byte[] key1 = "keyBBCF0".getBytes(UTF_8); + byte[] key2 = "keyBBCF1".getBytes(UTF_8); + db.put(columnFamilyHandleList.get(0), key1, "valueBBCF0".getBytes(UTF_8)); + db.put(columnFamilyHandleList.get(1), key2, "valueBBCF1".getBytes(UTF_8)); + + ByteBuffer key1Buff = ByteBuffer.allocateDirect(key1.length); + key1Buff.put(key1); + key1Buff.flip(); + + ByteBuffer key2Buff = ByteBuffer.allocateDirect(key2.length); + key2Buff.put(key2); + key2Buff.flip(); + + assertThat(db.keyExists(columnFamilyHandleList.get(0), key1Buff)).isTrue(); + assertThat(db.keyExists(columnFamilyHandleList.get(0), key2Buff)).isFalse(); + + assertThat(db.keyExists(columnFamilyHandleList.get(1), key1Buff)).isFalse(); + assertThat(db.keyExists(columnFamilyHandleList.get(1), key2Buff)).isTrue(); + } + + @Test + public void keyExistsDirectByteBufferColumnFamilyReadOptions() throws RocksDBException { + try (final ReadOptions readOptions = new ReadOptions()) { + byte[] key1 = "keyBBCF0".getBytes(UTF_8); + byte[] key2 = "keyBBCF1".getBytes(UTF_8); + db.put(columnFamilyHandleList.get(0), key1, "valueBBCF0".getBytes(UTF_8)); + db.put(columnFamilyHandleList.get(1), key2, "valueBBCF1".getBytes(UTF_8)); + + ByteBuffer key1Buff = ByteBuffer.allocateDirect(key1.length); + key1Buff.put(key1); + key1Buff.flip(); + + ByteBuffer key2Buff = ByteBuffer.allocateDirect(key2.length); + key2Buff.put(key2); + key2Buff.flip(); + + assertThat(db.keyExists(columnFamilyHandleList.get(0), readOptions, key1Buff)).isTrue(); + assertThat(db.keyExists(columnFamilyHandleList.get(0), readOptions, key2Buff)).isFalse(); + + assertThat(db.keyExists(columnFamilyHandleList.get(1), readOptions, key1Buff)).isFalse(); + assertThat(db.keyExists(columnFamilyHandleList.get(1), readOptions, key2Buff)).isTrue(); + } + } + + @Test + public void keyExistsDirectReadOptions() throws RocksDBException { + try (final ReadOptions readOptions = new ReadOptions()) { + byte[] key = "key1".getBytes(UTF_8); + db.put(key, "value".getBytes(UTF_8)); + ByteBuffer buff = ByteBuffer.allocateDirect(key.length); + buff.put(key); + buff.flip(); + boolean exists = db.keyExists(readOptions, key); + assertThat(exists).isTrue(); + buff.clear(); + + buff.put("key2".getBytes(UTF_8)); + buff.flip(); + exists = db.keyExists("key2".getBytes(UTF_8)); + assertThat(exists).isFalse(); + } + } +} diff --git a/java/src/test/java/org/rocksdb/KeyMayExistTest.java b/java/src/test/java/org/forstdb/KeyMayExistTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/KeyMayExistTest.java rename to java/src/test/java/org/forstdb/KeyMayExistTest.java index 3f3bec6ba..387a238a1 100644 --- a/java/src/test/java/org/rocksdb/KeyMayExistTest.java +++ b/java/src/test/java/org/forstdb/KeyMayExistTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; @@ -261,10 +261,12 @@ public void keyMayExistBB() throws RocksDBException { keyBuffer.flip(); assertThat(db.keyMayExist(keyBuffer)).isEqualTo(true); + keyBuffer.flip(); final ByteBuffer valueBuffer = ByteBuffer.allocateDirect(value.length + 24); valueBuffer.position(12); KeyMayExist keyMayExist = db.keyMayExist(keyBuffer, valueBuffer); + keyBuffer.flip(); assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); assertThat(keyMayExist.valueLength).isEqualTo(value.length); assertThat(valueBuffer.position()).isEqualTo(12); @@ -303,10 +305,11 @@ public void keyMayExistBBReadOptions() throws RocksDBException { try (final ReadOptions readOptions = new ReadOptions()) { assertThat(db.keyMayExist(readOptions, keyBuffer)).isEqualTo(true); - + keyBuffer.flip(); final ByteBuffer valueBuffer = ByteBuffer.allocateDirect(value.length + 24); valueBuffer.position(12); KeyMayExist keyMayExist = db.keyMayExist(readOptions, keyBuffer, valueBuffer); + keyBuffer.flip(); assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); assertThat(keyMayExist.valueLength).isEqualTo(value.length); assertThat(valueBuffer.position()).isEqualTo(12); @@ -318,6 +321,7 @@ public void keyMayExistBBReadOptions() throws RocksDBException { valueBuffer.limit(value.length + 24); valueBuffer.position(25); keyMayExist = db.keyMayExist(readOptions, keyBuffer, valueBuffer); + keyBuffer.flip(); assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); assertThat(keyMayExist.valueLength).isEqualTo(value.length); assertThat(valueBuffer.position()).isEqualTo(25); @@ -362,7 +366,9 @@ public void keyMayExistBBCF() throws RocksDBException { keyBuffer.flip(); assertThat(db.keyMayExist(keyBuffer)).isEqualTo(true); + keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer)).isEqualTo(false); + keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(0), keyBuffer)).isEqualTo(true); // 1 is just a CF @@ -372,8 +378,11 @@ public void keyMayExistBBCF() throws RocksDBException { keyBuffer.flip(); assertThat(db.keyMayExist(keyBuffer)).isEqualTo(false); + keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer)).isEqualTo(true); + keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(0), keyBuffer)).isEqualTo(false); + keyBuffer.flip(); exceptionRule.expect(AssertionError.class); exceptionRule.expectMessage( @@ -395,8 +404,10 @@ public void keyMayExistBBCFReadOptions() throws RocksDBException { try (final ReadOptions readOptions = new ReadOptions()) { assertThat(db.keyMayExist(keyBuffer)).isEqualTo(true); + keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer)) .isEqualTo(false); + keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(0), readOptions, keyBuffer)) .isEqualTo(true); @@ -407,8 +418,10 @@ public void keyMayExistBBCFReadOptions() throws RocksDBException { keyBuffer.flip(); assertThat(db.keyMayExist(readOptions, keyBuffer)).isEqualTo(false); + keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer)) .isEqualTo(true); + keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(0), readOptions, keyBuffer)) .isEqualTo(false); @@ -432,10 +445,11 @@ public void keyMayExistBBCFOffset() throws RocksDBException { keyBuffer.flip(); assertThat(db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer)).isEqualTo(true); - + keyBuffer.flip(); final ByteBuffer valueBuffer = ByteBuffer.allocateDirect(value.length + 24); valueBuffer.position(12); KeyMayExist keyMayExist = db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer, valueBuffer); + keyBuffer.flip(); assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); assertThat(keyMayExist.valueLength).isEqualTo(value.length); assertThat(valueBuffer.position()).isEqualTo(12); @@ -474,11 +488,12 @@ public void keyMayExistBBCFOffsetReadOptions() throws RocksDBException { try (final ReadOptions readOptions = new ReadOptions()) { assertThat(db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer)) .isEqualTo(true); - + keyBuffer.flip(); final ByteBuffer valueBuffer = ByteBuffer.allocateDirect(value.length + 24); valueBuffer.position(12); KeyMayExist keyMayExist = db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer, valueBuffer); + keyBuffer.flip(); assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); assertThat(keyMayExist.valueLength).isEqualTo(value.length); assertThat(valueBuffer.position()).isEqualTo(12); @@ -491,6 +506,7 @@ public void keyMayExistBBCFOffsetReadOptions() throws RocksDBException { valueBuffer.position(25); keyMayExist = db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer, valueBuffer); + keyBuffer.flip(); assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); assertThat(keyMayExist.valueLength).isEqualTo(value.length); assertThat(valueBuffer.position()).isEqualTo(25); diff --git a/java/src/test/java/org/rocksdb/LRUCacheTest.java b/java/src/test/java/org/forstdb/LRUCacheTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/LRUCacheTest.java rename to java/src/test/java/org/forstdb/LRUCacheTest.java index 4d194e712..e2782720e 100644 --- a/java/src/test/java/org/rocksdb/LRUCacheTest.java +++ b/java/src/test/java/org/forstdb/LRUCacheTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/LoggerTest.java b/java/src/test/java/org/forstdb/LoggerTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/LoggerTest.java rename to java/src/test/java/org/forstdb/LoggerTest.java index b6a7be55e..f91a6c260 100644 --- a/java/src/test/java/org/rocksdb/LoggerTest.java +++ b/java/src/test/java/org/forstdb/LoggerTest.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/MemTableTest.java b/java/src/test/java/org/forstdb/MemTableTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MemTableTest.java rename to java/src/test/java/org/forstdb/MemTableTest.java index 6ebf9ef51..2cf0ff0ec 100644 --- a/java/src/test/java/org/rocksdb/MemTableTest.java +++ b/java/src/test/java/org/forstdb/MemTableTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/MemoryUtilTest.java b/java/src/test/java/org/forstdb/MemoryUtilTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MemoryUtilTest.java rename to java/src/test/java/org/forstdb/MemoryUtilTest.java index bfdcb9fe1..555463706 100644 --- a/java/src/test/java/org/rocksdb/MemoryUtilTest.java +++ b/java/src/test/java/org/forstdb/MemoryUtilTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/forstdb/MergeCFVariantsTest.java b/java/src/test/java/org/forstdb/MergeCFVariantsTest.java new file mode 100644 index 000000000..5f3206fc6 --- /dev/null +++ b/java/src/test/java/org/forstdb/MergeCFVariantsTest.java @@ -0,0 +1,126 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.forstdb.MergeTest.longFromByteArray; +import static org.forstdb.MergeTest.longToByteArray; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class MergeCFVariantsTest { + @FunctionalInterface + interface FunctionMerge { + public void apply(PDatabase db, PColumnFamilyHandle one, PLeft two, PRight three) + throws RocksDBException; + } + + @Parameterized.Parameters + public static List> data() { + return Arrays.asList(RocksDB::merge, + (db, cfh, left, right) + -> db.merge(cfh, new WriteOptions(), left, right), + (db, cfh, left, right) + -> { + final byte[] left0 = + ("1234567" + new String(left, StandardCharsets.UTF_8) + "890").getBytes(); + final byte[] right0 = + ("1234" + new String(right, StandardCharsets.UTF_8) + "567890ab").getBytes(); + db.merge(cfh, left0, 7, left.length, right0, 4, right.length); + }, + (db, cfh, left, right) + -> { + final byte[] left0 = + ("1234567" + new String(left, StandardCharsets.UTF_8) + "890").getBytes(); + final byte[] right0 = + ("1234" + new String(right, StandardCharsets.UTF_8) + "567890ab").getBytes(); + db.merge(cfh, new WriteOptions(), left0, 7, left.length, right0, 4, right.length); + }, + (db, cfh, left, right) + -> { + final ByteBuffer bbLeft = ByteBuffer.allocateDirect(100); + final ByteBuffer bbRight = ByteBuffer.allocateDirect(100); + bbLeft.put(left).flip(); + bbRight.put(right).flip(); + db.merge(cfh, new WriteOptions(), bbLeft, bbRight); + }, + (db, cfh, left, right) -> { + final ByteBuffer bbLeft = ByteBuffer.allocate(100); + final ByteBuffer bbRight = ByteBuffer.allocate(100); + bbLeft.put(left).flip(); + bbRight.put(right).flip(); + db.merge(cfh, new WriteOptions(), bbLeft, bbRight); + }); + } + + @Parameterized.Parameter + public FunctionMerge mergeFunction; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void cFUInt64AddOperatorOption() throws InterruptedException, RocksDBException { + try (final UInt64AddOperator uint64AddOperator = new UInt64AddOperator(); + final ColumnFamilyOptions cfOpt1 = + new ColumnFamilyOptions().setMergeOperator(uint64AddOperator); + final ColumnFamilyOptions cfOpt2 = + new ColumnFamilyOptions().setMergeOperator(uint64AddOperator)) { + final List cfDescriptors = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpt1), + new ColumnFamilyDescriptor("new_cf".getBytes(), cfOpt2)); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions opt = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open( + opt, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList)) { + try { + // writing (long)100 under key + db.put(columnFamilyHandleList.get(1), "cfkey".getBytes(), longToByteArray(100)); + // merge (long)1 under key + mergeFunction.apply( + db, columnFamilyHandleList.get(1), "cfkey".getBytes(), longToByteArray(1)); + byte[] value = db.get(columnFamilyHandleList.get(1), "cfkey".getBytes()); + long longValue = longFromByteArray(value); + + // Test also with createColumnFamily + try (final ColumnFamilyOptions cfHandleOpts = + new ColumnFamilyOptions().setMergeOperator(uint64AddOperator); + final ColumnFamilyHandle cfHandle = db.createColumnFamily( + new ColumnFamilyDescriptor("new_cf2".getBytes(), cfHandleOpts))) { + // writing (long)200 under cfkey2 + db.put(cfHandle, "cfkey2".getBytes(), longToByteArray(200)); + // merge (long)50 under cfkey2 + db.merge(cfHandle, new WriteOptions(), "cfkey2".getBytes(), longToByteArray(50)); + value = db.get(cfHandle, "cfkey2".getBytes()); + long longValueTmpCf = longFromByteArray(value); + + assertThat(longValue).isEqualTo(101); + assertThat(longValueTmpCf).isEqualTo(250); + } + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + } +} diff --git a/java/src/test/java/org/rocksdb/MergeTest.java b/java/src/test/java/org/forstdb/MergeTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MergeTest.java rename to java/src/test/java/org/forstdb/MergeTest.java index f99ac49d3..2612027a6 100644 --- a/java/src/test/java/org/rocksdb/MergeTest.java +++ b/java/src/test/java/org/forstdb/MergeTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; @@ -45,14 +45,14 @@ public void stringOption() } } - private byte[] longToByteArray(final long l) { + static byte[] longToByteArray(final long l) { final ByteBuffer buf = ByteBuffer.allocate(Long.SIZE / Byte.SIZE).order(ByteOrder.LITTLE_ENDIAN); buf.putLong(l); return buf.array(); } - private long longFromByteArray(final byte[] a) { + static long longFromByteArray(final byte[] a) { final ByteBuffer buf = ByteBuffer.allocate(Long.SIZE / Byte.SIZE).order(ByteOrder.LITTLE_ENDIAN); buf.put(a); diff --git a/java/src/test/java/org/forstdb/MergeVariantsTest.java b/java/src/test/java/org/forstdb/MergeVariantsTest.java new file mode 100644 index 000000000..bcf9edc1a --- /dev/null +++ b/java/src/test/java/org/forstdb/MergeVariantsTest.java @@ -0,0 +1,95 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.forstdb.MergeTest.longFromByteArray; +import static org.forstdb.MergeTest.longToByteArray; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class MergeVariantsTest { + @FunctionalInterface + interface FunctionMerge { + public void apply(PDatabase db, PLeft two, PRight three) throws RocksDBException; + } + + @Parameterized.Parameters + public static List> data() { + return Arrays.asList(RocksDB::merge, + (db, left, right) + -> db.merge(new WriteOptions(), left, right), + (db, left, right) + -> { + final byte[] left0 = + ("1234567" + new String(left, StandardCharsets.UTF_8) + "890").getBytes(); + final byte[] right0 = + ("1234" + new String(right, StandardCharsets.UTF_8) + "567890ab").getBytes(); + db.merge(left0, 7, left.length, right0, 4, right.length); + }, + (db, left, right) + -> { + final byte[] left0 = + ("1234567" + new String(left, StandardCharsets.UTF_8) + "890").getBytes(); + final byte[] right0 = + ("1234" + new String(right, StandardCharsets.UTF_8) + "567890ab").getBytes(); + db.merge(new WriteOptions(), left0, 7, left.length, right0, 4, right.length); + }, + (db, left, right) + -> { + final ByteBuffer bbLeft = ByteBuffer.allocateDirect(100); + final ByteBuffer bbRight = ByteBuffer.allocateDirect(100); + bbLeft.put(left).flip(); + bbRight.put(right).flip(); + db.merge(new WriteOptions(), bbLeft, bbRight); + }, + (db, left, right) -> { + final ByteBuffer bbLeft = ByteBuffer.allocate(100); + final ByteBuffer bbRight = ByteBuffer.allocate(100); + bbLeft.put(left).flip(); + bbRight.put(right).flip(); + db.merge(new WriteOptions(), bbLeft, bbRight); + }); + } + + @Parameterized.Parameter + public MergeVariantsTest.FunctionMerge mergeFunction; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void uint64AddOperatorOption() throws InterruptedException, RocksDBException { + try (final UInt64AddOperator uint64AddOperator = new UInt64AddOperator(); + final Options opt = + new Options().setCreateIfMissing(true).setMergeOperator(uint64AddOperator); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + // Writing (long)100 under key + db.put("key".getBytes(), longToByteArray(100)); + + // Writing (long)1 under key + mergeFunction.apply(db, "key".getBytes(), longToByteArray(1)); + + final byte[] value = db.get("key".getBytes()); + final long longValue = longFromByteArray(value); + + assertThat(longValue).isEqualTo(101); + } + } +} diff --git a/java/src/test/java/org/rocksdb/MixedOptionsTest.java b/java/src/test/java/org/forstdb/MixedOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MixedOptionsTest.java rename to java/src/test/java/org/forstdb/MixedOptionsTest.java index 4e17d04ef..4a1b40d47 100644 --- a/java/src/test/java/org/rocksdb/MixedOptionsTest.java +++ b/java/src/test/java/org/forstdb/MixedOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/MultiColumnRegressionTest.java b/java/src/test/java/org/forstdb/MultiColumnRegressionTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MultiColumnRegressionTest.java rename to java/src/test/java/org/forstdb/MultiColumnRegressionTest.java index 6087b0260..7902a4af7 100644 --- a/java/src/test/java/org/rocksdb/MultiColumnRegressionTest.java +++ b/java/src/test/java/org/forstdb/MultiColumnRegressionTest.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/MultiGetManyKeysTest.java b/java/src/test/java/org/forstdb/MultiGetManyKeysTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MultiGetManyKeysTest.java rename to java/src/test/java/org/forstdb/MultiGetManyKeysTest.java index e66eef622..c65e4fd1d 100644 --- a/java/src/test/java/org/rocksdb/MultiGetManyKeysTest.java +++ b/java/src/test/java/org/forstdb/MultiGetManyKeysTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/MultiGetTest.java b/java/src/test/java/org/forstdb/MultiGetTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MultiGetTest.java rename to java/src/test/java/org/forstdb/MultiGetTest.java index c391d81f6..809ac8ca2 100644 --- a/java/src/test/java/org/rocksdb/MultiGetTest.java +++ b/java/src/test/java/org/forstdb/MultiGetTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; @@ -15,7 +15,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.TestUtil; +import org.forstdb.util.TestUtil; public class MultiGetTest { @ClassRule diff --git a/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java b/java/src/test/java/org/forstdb/MutableColumnFamilyOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java rename to java/src/test/java/org/forstdb/MutableColumnFamilyOptionsTest.java index d858a150d..746c219e7 100644 --- a/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java +++ b/java/src/test/java/org/forstdb/MutableColumnFamilyOptionsTest.java @@ -2,10 +2,10 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; -import org.rocksdb.MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder; +import org.forstdb.MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder; import java.util.NoSuchElementException; diff --git a/java/src/test/java/org/rocksdb/MutableDBOptionsTest.java b/java/src/test/java/org/forstdb/MutableDBOptionsTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/MutableDBOptionsTest.java rename to java/src/test/java/org/forstdb/MutableDBOptionsTest.java index 063a8de38..9298181d3 100644 --- a/java/src/test/java/org/rocksdb/MutableDBOptionsTest.java +++ b/java/src/test/java/org/forstdb/MutableDBOptionsTest.java @@ -2,10 +2,10 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; -import org.rocksdb.MutableDBOptions.MutableDBOptionsBuilder; +import org.forstdb.MutableDBOptions.MutableDBOptionsBuilder; import java.util.NoSuchElementException; diff --git a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java b/java/src/test/java/org/forstdb/MutableOptionsGetSetTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java rename to java/src/test/java/org/forstdb/MutableOptionsGetSetTest.java index 6db940619..4628dd417 100644 --- a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java +++ b/java/src/test/java/org/forstdb/MutableOptionsGetSetTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/NativeComparatorWrapperTest.java b/java/src/test/java/org/forstdb/NativeComparatorWrapperTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/NativeComparatorWrapperTest.java rename to java/src/test/java/org/forstdb/NativeComparatorWrapperTest.java index 1e0ded816..a5aaac158 100644 --- a/java/src/test/java/org/rocksdb/NativeComparatorWrapperTest.java +++ b/java/src/test/java/org/forstdb/NativeComparatorWrapperTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.junit.Assert.assertEquals; diff --git a/java/src/test/java/org/rocksdb/NativeLibraryLoaderTest.java b/java/src/test/java/org/forstdb/NativeLibraryLoaderTest.java similarity index 92% rename from java/src/test/java/org/rocksdb/NativeLibraryLoaderTest.java rename to java/src/test/java/org/forstdb/NativeLibraryLoaderTest.java index 6b954f67e..4463f51f5 100644 --- a/java/src/test/java/org/rocksdb/NativeLibraryLoaderTest.java +++ b/java/src/test/java/org/forstdb/NativeLibraryLoaderTest.java @@ -2,12 +2,12 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.Environment; +import org.forstdb.util.Environment; import java.io.File; import java.io.IOException; @@ -25,7 +25,7 @@ public void tempFolder() throws IOException { NativeLibraryLoader.getInstance().loadLibraryFromJarToTemp( temporaryFolder.getRoot().getAbsolutePath()); final Path path = Paths.get(temporaryFolder.getRoot().getAbsolutePath(), - Environment.getJniLibraryFileName("rocksdb")); + Environment.getJniLibraryFileName("forst")); assertThat(Files.exists(path)).isTrue(); assertThat(Files.isReadable(path)).isTrue(); } diff --git a/java/src/test/java/org/rocksdb/OptimisticTransactionDBTest.java b/java/src/test/java/org/forstdb/OptimisticTransactionDBTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/OptimisticTransactionDBTest.java rename to java/src/test/java/org/forstdb/OptimisticTransactionDBTest.java index 519b70b1d..20b5d442c 100644 --- a/java/src/test/java/org/rocksdb/OptimisticTransactionDBTest.java +++ b/java/src/test/java/org/forstdb/OptimisticTransactionDBTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Rule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/OptimisticTransactionOptionsTest.java b/java/src/test/java/org/forstdb/OptimisticTransactionOptionsTest.java similarity index 94% rename from java/src/test/java/org/rocksdb/OptimisticTransactionOptionsTest.java rename to java/src/test/java/org/forstdb/OptimisticTransactionOptionsTest.java index ef656b958..2190dc963 100644 --- a/java/src/test/java/org/rocksdb/OptimisticTransactionOptionsTest.java +++ b/java/src/test/java/org/forstdb/OptimisticTransactionOptionsTest.java @@ -3,10 +3,10 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; -import org.rocksdb.util.BytewiseComparator; +import org.forstdb.util.BytewiseComparator; import java.util.Random; diff --git a/java/src/test/java/org/rocksdb/OptimisticTransactionTest.java b/java/src/test/java/org/forstdb/OptimisticTransactionTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/OptimisticTransactionTest.java rename to java/src/test/java/org/forstdb/OptimisticTransactionTest.java index d2f92e1ff..c8a5f7997 100644 --- a/java/src/test/java/org/rocksdb/OptimisticTransactionTest.java +++ b/java/src/test/java/org/forstdb/OptimisticTransactionTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.*; @@ -373,12 +373,13 @@ public OptimisticTransactionDBContainer startDb() .setCreateIfMissing(true) .setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions defaultColumnFamilyOptions = new ColumnFamilyOptions(); + defaultColumnFamilyOptions.setMergeOperator(new StringAppendOperator("++")); final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); - final List columnFamilyDescriptors = - Arrays.asList( - new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), - new ColumnFamilyDescriptor(TXN_TEST_COLUMN_FAMILY, - columnFamilyOptions)); + columnFamilyOptions.setMergeOperator(new StringAppendOperator("**")); + final List columnFamilyDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultColumnFamilyOptions), + new ColumnFamilyDescriptor(TXN_TEST_COLUMN_FAMILY, columnFamilyOptions)); final List columnFamilyHandles = new ArrayList<>(); final OptimisticTransactionDB optimisticTxnDb; diff --git a/java/src/test/java/org/rocksdb/OptionsTest.java b/java/src/test/java/org/forstdb/OptionsTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/OptionsTest.java rename to java/src/test/java/org/forstdb/OptionsTest.java index e1a7f8c27..898aefc26 100644 --- a/java/src/test/java/org/rocksdb/OptionsTest.java +++ b/java/src/test/java/org/forstdb/OptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.*; @@ -13,7 +13,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.junit.ClassRule; import org.junit.Test; -import org.rocksdb.test.RemoveEmptyValueCompactionFilterFactory; +import org.forstdb.test.RemoveEmptyValueCompactionFilterFactory; public class OptionsTest { @@ -699,6 +699,7 @@ public void setWriteBufferManagerWithAllowStall() throws RocksDBException { } } + @SuppressWarnings("deprecated") @Test public void accessHintOnCompactionStart() { try (final Options opt = new Options()) { @@ -1452,6 +1453,16 @@ public void skipCheckingSstFileSizesOnDbOpen() { } } + @Test + public void memtableMaxRangeDeletions() { + try (final Options options = new Options()) { + assertThat(options.memtableMaxRangeDeletions()).isEqualTo(0); + final int val = 32; + assertThat(options.setMemtableMaxRangeDeletions(val)).isEqualTo(options); + assertThat(options.memtableMaxRangeDeletions()).isEqualTo(val); + } + } + @Test public void eventListeners() { final AtomicBoolean wasCalled1 = new AtomicBoolean(); diff --git a/java/src/test/java/org/forstdb/OptionsUtilTest.java b/java/src/test/java/org/forstdb/OptionsUtilTest.java new file mode 100644 index 000000000..f9725efd7 --- /dev/null +++ b/java/src/test/java/org/forstdb/OptionsUtilTest.java @@ -0,0 +1,378 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.*; + +import static org.assertj.core.api.Assertions.assertThat; + +public class OptionsUtilTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void loadLatestOptions() throws RocksDBException { + verifyOptions(new LoaderUnderTest() { + @Override + List loadOptions(final String dbPath, final DBOptions dbOptions) + throws RocksDBException { + try (final ConfigOptions configOptions = new ConfigOptions() + .setIgnoreUnknownOptions(false) + .setInputStringsEscaped(true) + .setEnv(Env.getDefault())) { + final List cfDescs = new ArrayList<>(); + OptionsUtil.loadLatestOptions(configOptions, dbPath, dbOptions, cfDescs); + return cfDescs; + } + } + }); + } + + @Test + public void loadOptionsFromFile() throws RocksDBException { + verifyOptions(new LoaderUnderTest() { + @Override + List loadOptions(final String dbPath, final DBOptions dbOptions) + throws RocksDBException { + try (final ConfigOptions configOptions = new ConfigOptions() + .setIgnoreUnknownOptions(false) + .setInputStringsEscaped(true) + .setEnv(Env.getDefault())) { + final List cfDescs = new ArrayList<>(); + final String path = + dbPath + "/" + OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); + OptionsUtil.loadOptionsFromFile(configOptions, path, dbOptions, cfDescs); + return cfDescs; + } + } + }); + } + + @Test + public void loadLatestTableFormatOptions() throws RocksDBException { + verifyTableFormatOptions(new LoaderUnderTest() { + @Override + List loadOptions(final String dbPath, final DBOptions dbOptions) + throws RocksDBException { + try (final ConfigOptions configOptions = new ConfigOptions() + .setIgnoreUnknownOptions(false) + .setInputStringsEscaped(true) + .setEnv(Env.getDefault())) { + final List cfDescs = new ArrayList<>(); + OptionsUtil.loadLatestOptions(configOptions, dbPath, dbOptions, cfDescs); + return cfDescs; + } + } + }); + } + + @Test + public void loadLatestTableFormatOptions2() throws RocksDBException { + verifyTableFormatOptions(new LoaderUnderTest() { + @Override + List loadOptions(final String dbPath, final DBOptions dbOptions) + throws RocksDBException { + try (final ConfigOptions configOptions = new ConfigOptions() + .setIgnoreUnknownOptions(false) + .setInputStringsEscaped(true) + .setEnv(Env.getDefault())) { + final List cfDescs = new ArrayList<>(); + OptionsUtil.loadLatestOptions(configOptions, dbPath, dbOptions, cfDescs); + return cfDescs; + } + } + }); + } + + @Test + public void loadLatestTableFormatOptions3() throws RocksDBException { + verifyTableFormatOptions(new LoaderUnderTest() { + @Override + List loadOptions(final String dbPath, final DBOptions dbOptions) + throws RocksDBException { + final List cfDescs = new ArrayList<>(); + OptionsUtil.loadLatestOptions(new ConfigOptions(), dbPath, dbOptions, cfDescs); + return cfDescs; + } + }); + } + + @Test + public void loadTableFormatOptionsFromFile() throws RocksDBException { + verifyTableFormatOptions(new LoaderUnderTest() { + @Override + List loadOptions(final String dbPath, final DBOptions dbOptions) + throws RocksDBException { + try (final ConfigOptions configOptions = new ConfigOptions() + .setIgnoreUnknownOptions(false) + .setInputStringsEscaped(true) + .setEnv(Env.getDefault())) { + final List cfDescs = new ArrayList<>(); + final String path = + dbPath + "/" + OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); + OptionsUtil.loadOptionsFromFile(configOptions, path, dbOptions, cfDescs); + return cfDescs; + } + } + }); + } + + @Test + public void loadTableFormatOptionsFromFile2() throws RocksDBException { + verifyTableFormatOptions(new LoaderUnderTest() { + @Override + List loadOptions(final String dbPath, final DBOptions dbOptions) + throws RocksDBException { + try (final ConfigOptions configOptions = new ConfigOptions() + .setIgnoreUnknownOptions(false) + .setInputStringsEscaped(true) + .setEnv(Env.getDefault())) { + final List cfDescs = new ArrayList<>(); + final String path = + dbPath + "/" + OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); + OptionsUtil.loadOptionsFromFile(configOptions, path, dbOptions, cfDescs); + return cfDescs; + } + } + }); + } + + @Test + public void loadTableFormatOptionsFromFile3() throws RocksDBException { + verifyTableFormatOptions(new LoaderUnderTest() { + @Override + List loadOptions(final String dbPath, final DBOptions dbOptions) + throws RocksDBException { + final List cfDescs = new ArrayList<>(); + final String path = + dbPath + "/" + OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); + OptionsUtil.loadOptionsFromFile(new ConfigOptions(), path, dbOptions, cfDescs); + return cfDescs; + } + }); + } + + @Test + public void getLatestOptionsFileName() throws RocksDBException { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db).isNotNull(); + } + + final String fName = OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); + assertThat(fName).isNotNull(); + assert (fName.startsWith("OPTIONS-")); + // System.out.println("latest options fileName: " + fName); + } + + static abstract class LoaderUnderTest { + abstract List loadOptions(final String path, final DBOptions dbOptions) + throws RocksDBException; + } + + private void verifyOptions(final LoaderUnderTest loaderUnderTest) throws RocksDBException { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + final Options options = new Options() + .setCreateIfMissing(true) + .setParanoidChecks(false) + .setMaxOpenFiles(478) + .setDelayedWriteRate(1234567L); + final ColumnFamilyOptions baseDefaultCFOpts = new ColumnFamilyOptions(); + final byte[] secondCFName = "new_cf".getBytes(); + final ColumnFamilyOptions baseSecondCFOpts = + new ColumnFamilyOptions() + .setWriteBufferSize(70 * 1024) + .setMaxWriteBufferNumber(7) + .setMaxBytesForLevelBase(53 * 1024 * 1024) + .setLevel0FileNumCompactionTrigger(3) + .setLevel0SlowdownWritesTrigger(51) + .setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); + + // Create a database with a new column family + try (final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db).isNotNull(); + + // create column family + try (final ColumnFamilyHandle columnFamilyHandle = + db.createColumnFamily(new ColumnFamilyDescriptor(secondCFName, baseSecondCFOpts))) { + assert(columnFamilyHandle != null); + } + } + + // Read the options back and verify + try (DBOptions dbOptions = new DBOptions()) { + final List cfDescs = loaderUnderTest.loadOptions(dbPath, dbOptions); + + assertThat(dbOptions.createIfMissing()).isEqualTo(options.createIfMissing()); + assertThat(dbOptions.paranoidChecks()).isEqualTo(options.paranoidChecks()); + assertThat(dbOptions.maxOpenFiles()).isEqualTo(options.maxOpenFiles()); + assertThat(dbOptions.delayedWriteRate()).isEqualTo(options.delayedWriteRate()); + + assertThat(cfDescs.size()).isEqualTo(2); + assertThat(cfDescs.get(0)).isNotNull(); + assertThat(cfDescs.get(1)).isNotNull(); + assertThat(cfDescs.get(0).getName()).isEqualTo(RocksDB.DEFAULT_COLUMN_FAMILY); + assertThat(cfDescs.get(1).getName()).isEqualTo(secondCFName); + + final ColumnFamilyOptions defaultCFOpts = cfDescs.get(0).getOptions(); + assertThat(defaultCFOpts.writeBufferSize()).isEqualTo(baseDefaultCFOpts.writeBufferSize()); + assertThat(defaultCFOpts.maxWriteBufferNumber()) + .isEqualTo(baseDefaultCFOpts.maxWriteBufferNumber()); + assertThat(defaultCFOpts.maxBytesForLevelBase()) + .isEqualTo(baseDefaultCFOpts.maxBytesForLevelBase()); + assertThat(defaultCFOpts.level0FileNumCompactionTrigger()) + .isEqualTo(baseDefaultCFOpts.level0FileNumCompactionTrigger()); + assertThat(defaultCFOpts.level0SlowdownWritesTrigger()) + .isEqualTo(baseDefaultCFOpts.level0SlowdownWritesTrigger()); + assertThat(defaultCFOpts.bottommostCompressionType()) + .isEqualTo(baseDefaultCFOpts.bottommostCompressionType()); + + final ColumnFamilyOptions secondCFOpts = cfDescs.get(1).getOptions(); + assertThat(secondCFOpts.writeBufferSize()).isEqualTo(baseSecondCFOpts.writeBufferSize()); + assertThat(secondCFOpts.maxWriteBufferNumber()) + .isEqualTo(baseSecondCFOpts.maxWriteBufferNumber()); + assertThat(secondCFOpts.maxBytesForLevelBase()) + .isEqualTo(baseSecondCFOpts.maxBytesForLevelBase()); + assertThat(secondCFOpts.level0FileNumCompactionTrigger()) + .isEqualTo(baseSecondCFOpts.level0FileNumCompactionTrigger()); + assertThat(secondCFOpts.level0SlowdownWritesTrigger()) + .isEqualTo(baseSecondCFOpts.level0SlowdownWritesTrigger()); + assertThat(secondCFOpts.bottommostCompressionType()) + .isEqualTo(baseSecondCFOpts.bottommostCompressionType()); + } + } + + private void verifyTableFormatOptions(final LoaderUnderTest loaderUnderTest) + throws RocksDBException { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + final Options options = new Options() + .setCreateIfMissing(true) + .setParanoidChecks(false) + .setMaxOpenFiles(478) + .setDelayedWriteRate(1234567L); + final ColumnFamilyOptions defaultCFOptions = new ColumnFamilyOptions(); + defaultCFOptions.setTableFormatConfig(new BlockBasedTableConfig()); + final byte[] altCFName = "alt_cf".getBytes(); + final ColumnFamilyOptions altCFOptions = + new ColumnFamilyOptions() + .setWriteBufferSize(70 * 1024) + .setMaxWriteBufferNumber(7) + .setMaxBytesForLevelBase(53 * 1024 * 1024) + .setLevel0FileNumCompactionTrigger(3) + .setLevel0SlowdownWritesTrigger(51) + .setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); + + final BlockBasedTableConfig altCFTableConfig = new BlockBasedTableConfig(); + altCFTableConfig.setCacheIndexAndFilterBlocks(true); + altCFTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(false); + altCFTableConfig.setPinL0FilterAndIndexBlocksInCache(true); + altCFTableConfig.setPinTopLevelIndexAndFilter(false); + altCFTableConfig.setIndexType(IndexType.kTwoLevelIndexSearch); + altCFTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash); + altCFTableConfig.setDataBlockHashTableUtilRatio(0.65); + altCFTableConfig.setChecksumType(ChecksumType.kxxHash64); + altCFTableConfig.setNoBlockCache(true); + altCFTableConfig.setBlockSize(35 * 1024); + altCFTableConfig.setBlockSizeDeviation(20); + altCFTableConfig.setBlockRestartInterval(12); + altCFTableConfig.setIndexBlockRestartInterval(6); + altCFTableConfig.setMetadataBlockSize(12 * 1024); + altCFTableConfig.setPartitionFilters(true); + altCFTableConfig.setOptimizeFiltersForMemory(true); + altCFTableConfig.setUseDeltaEncoding(false); + altCFTableConfig.setFilterPolicy(new BloomFilter(7.5)); + altCFTableConfig.setWholeKeyFiltering(false); + altCFTableConfig.setVerifyCompression(true); + altCFTableConfig.setReadAmpBytesPerBit(2); + altCFTableConfig.setFormatVersion(8); + altCFTableConfig.setEnableIndexCompression(false); + altCFTableConfig.setBlockAlign(true); + altCFTableConfig.setIndexShortening(IndexShorteningMode.kShortenSeparatorsAndSuccessor); + altCFTableConfig.setBlockCacheSize(3 * 1024 * 1024); + // Note cache objects are not set here, as they are not read back when reading config. + + altCFOptions.setTableFormatConfig(altCFTableConfig); + + // Create a database with a new column family + try (final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db).isNotNull(); + + // create column family + try (final ColumnFamilyHandle columnFamilyHandle = + db.createColumnFamily(new ColumnFamilyDescriptor(altCFName, altCFOptions))) { + assert (columnFamilyHandle != null); + } + } + + // Read the options back and verify + final DBOptions dbOptions = new DBOptions(); + final List cfDescs = loaderUnderTest.loadOptions(dbPath, dbOptions); + + assertThat(dbOptions.createIfMissing()).isEqualTo(options.createIfMissing()); + assertThat(dbOptions.paranoidChecks()).isEqualTo(options.paranoidChecks()); + assertThat(dbOptions.maxOpenFiles()).isEqualTo(options.maxOpenFiles()); + assertThat(dbOptions.delayedWriteRate()).isEqualTo(options.delayedWriteRate()); + + assertThat(cfDescs.size()).isEqualTo(2); + assertThat(cfDescs.get(0)).isNotNull(); + assertThat(cfDescs.get(1)).isNotNull(); + assertThat(cfDescs.get(0).getName()).isEqualTo(RocksDB.DEFAULT_COLUMN_FAMILY); + assertThat(cfDescs.get(1).getName()).isEqualTo(altCFName); + + verifyBlockBasedTableConfig( + cfDescs.get(0).getOptions().tableFormatConfig(), new BlockBasedTableConfig()); + verifyBlockBasedTableConfig(cfDescs.get(1).getOptions().tableFormatConfig(), altCFTableConfig); + } + + private void verifyBlockBasedTableConfig( + final TableFormatConfig actualTableConfig, final BlockBasedTableConfig expected) { + assertThat(actualTableConfig).isNotNull(); + assertThat(actualTableConfig).isInstanceOf(BlockBasedTableConfig.class); + final BlockBasedTableConfig actual = (BlockBasedTableConfig) actualTableConfig; + assertThat(actual.cacheIndexAndFilterBlocks()).isEqualTo(expected.cacheIndexAndFilterBlocks()); + assertThat(actual.cacheIndexAndFilterBlocksWithHighPriority()) + .isEqualTo(expected.cacheIndexAndFilterBlocksWithHighPriority()); + assertThat(actual.pinL0FilterAndIndexBlocksInCache()) + .isEqualTo(expected.pinL0FilterAndIndexBlocksInCache()); + assertThat(actual.indexType()).isEqualTo(expected.indexType()); + assertThat(actual.dataBlockIndexType()).isEqualTo(expected.dataBlockIndexType()); + assertThat(actual.dataBlockHashTableUtilRatio()) + .isEqualTo(expected.dataBlockHashTableUtilRatio()); + assertThat(actual.checksumType()).isEqualTo(expected.checksumType()); + assertThat(actual.noBlockCache()).isEqualTo(expected.noBlockCache()); + assertThat(actual.blockSize()).isEqualTo(expected.blockSize()); + assertThat(actual.blockSizeDeviation()).isEqualTo(expected.blockSizeDeviation()); + assertThat(actual.blockRestartInterval()).isEqualTo(expected.blockRestartInterval()); + assertThat(actual.indexBlockRestartInterval()).isEqualTo(expected.indexBlockRestartInterval()); + assertThat(actual.metadataBlockSize()).isEqualTo(expected.metadataBlockSize()); + assertThat(actual.partitionFilters()).isEqualTo(expected.partitionFilters()); + assertThat(actual.optimizeFiltersForMemory()).isEqualTo(expected.optimizeFiltersForMemory()); + assertThat(actual.wholeKeyFiltering()).isEqualTo(expected.wholeKeyFiltering()); + assertThat(actual.verifyCompression()).isEqualTo(expected.verifyCompression()); + assertThat(actual.readAmpBytesPerBit()).isEqualTo(expected.readAmpBytesPerBit()); + assertThat(actual.formatVersion()).isEqualTo(expected.formatVersion()); + assertThat(actual.enableIndexCompression()).isEqualTo(expected.enableIndexCompression()); + assertThat(actual.blockAlign()).isEqualTo(expected.blockAlign()); + assertThat(actual.indexShortening()).isEqualTo(expected.indexShortening()); + if (expected.filterPolicy() == null) { + assertThat(actual.filterPolicy()).isNull(); + } else { + assertThat(expected.filterPolicy().equals(actual.filterPolicy())); + } + + // not currently persisted - always true when read from options + // this test will fail, and need repaired, if and when "useDeltaEncoding" is persisted. + assertThat(actual.useDeltaEncoding()).isEqualTo(true); + } +} diff --git a/java/src/test/java/org/forstdb/PerfContextTest.java b/java/src/test/java/org/forstdb/PerfContextTest.java new file mode 100644 index 000000000..84bf3fa92 --- /dev/null +++ b/java/src/test/java/org/forstdb/PerfContextTest.java @@ -0,0 +1,102 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.beans.Introspector; +import java.beans.PropertyDescriptor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.*; +import org.junit.rules.TemporaryFolder; + +public class PerfContextTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + List cfDescriptors; + List columnFamilyHandleList = new ArrayList<>(); + RocksDB db; + + @Before + public void before() throws RocksDBException { + cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + + db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList); + } + + @After + public void after() { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + db.close(); + } + + @Test + public void testReset() { + db.setPerfLevel(PerfLevel.ENABLE_TIME_AND_CPU_TIME_EXCEPT_FOR_MUTEX); + PerfContext ctx = db.getPerfContext(); + assertThat(ctx).isNotNull(); + ctx.reset(); + } + + /** + * Call all properties to check that we don't have problem with UnsatisfiedLinkError. + */ + @Test + public void testAllGetters() throws RocksDBException, IntrospectionException, + InvocationTargetException, IllegalAccessException { + db.setPerfLevel(PerfLevel.ENABLE_TIME_AND_CPU_TIME_EXCEPT_FOR_MUTEX); + db.put("key".getBytes(), "value".getBytes()); + db.compactRange(); + db.get("key".getBytes()); + PerfContext ctx = db.getPerfContext(); + + BeanInfo info = Introspector.getBeanInfo(ctx.getClass(), RocksObject.class); + for (PropertyDescriptor property : info.getPropertyDescriptors()) { + if (property.getReadMethod() != null) { + Object result = property.getReadMethod().invoke(ctx); + assertThat(result).isNotNull(); + assertThat(result).isInstanceOf(Long.class); + } + } + } + + @Test + public void testGetBlockReadCpuTime() throws RocksDBException { + db.setPerfLevel(PerfLevel.ENABLE_TIME_AND_CPU_TIME_EXCEPT_FOR_MUTEX); + db.put("key".getBytes(), "value".getBytes()); + db.compactRange(); + db.get("key".getBytes()); + PerfContext ctx = db.getPerfContext(); + assertThat(ctx).isNotNull(); + assertThat(ctx.getBlockReadCpuTime()).isGreaterThan(0); + } + + @Test + public void testGetPostProcessTime() throws RocksDBException { + db.setPerfLevel(PerfLevel.ENABLE_TIME_AND_CPU_TIME_EXCEPT_FOR_MUTEX); + db.put("key".getBytes(), "value".getBytes()); + db.compactRange(); + db.get("key".getBytes()); + PerfContext ctx = db.getPerfContext(); + assertThat(ctx).isNotNull(); + assertThat(ctx.getPostProcessTime()).isGreaterThan(0); + } +} diff --git a/java/src/test/java/org/forstdb/PerfLevelTest.java b/java/src/test/java/org/forstdb/PerfLevelTest.java new file mode 100644 index 000000000..d3c8d6cc0 --- /dev/null +++ b/java/src/test/java/org/forstdb/PerfLevelTest.java @@ -0,0 +1,65 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.forstdb.PerfLevel.*; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.*; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +public class PerfLevelTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + List cfDescriptors; + List columnFamilyHandleList = new ArrayList<>(); + RocksDB db; + + @Before + public void before() throws RocksDBException { + cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + + db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList); + } + + @After + public void after() { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + db.close(); + } + @Test + public void testForInvalidValues() { + assertThatThrownBy(() -> db.setPerfLevel(UNINITIALIZED)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> db.setPerfLevel(OUT_OF_BOUNDS)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testAllPerfLevels() { + for (PerfLevel level : new PerfLevel[] {DISABLE, ENABLE_COUNT, ENABLE_TIME_EXCEPT_FOR_MUTEX, + ENABLE_TIME_AND_CPU_TIME_EXCEPT_FOR_MUTEX, ENABLE_TIME}) { + db.setPerfLevel(level); + assertThat(db.getPerfLevel()).isEqualTo(level); + } + db.setPerfLevel(DISABLE); + } +} diff --git a/java/src/test/java/org/rocksdb/PlainTableConfigTest.java b/java/src/test/java/org/forstdb/PlainTableConfigTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/PlainTableConfigTest.java rename to java/src/test/java/org/forstdb/PlainTableConfigTest.java index 827eb79f9..dca2c4777 100644 --- a/java/src/test/java/org/rocksdb/PlainTableConfigTest.java +++ b/java/src/test/java/org/forstdb/PlainTableConfigTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/PlatformRandomHelper.java b/java/src/test/java/org/forstdb/PlatformRandomHelper.java similarity index 98% rename from java/src/test/java/org/rocksdb/PlatformRandomHelper.java rename to java/src/test/java/org/forstdb/PlatformRandomHelper.java index 80ea4d197..ca1dbe7b7 100644 --- a/java/src/test/java/org/rocksdb/PlatformRandomHelper.java +++ b/java/src/test/java/org/forstdb/PlatformRandomHelper.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Random; diff --git a/java/src/test/java/org/forstdb/PutCFVariantsTest.java b/java/src/test/java/org/forstdb/PutCFVariantsTest.java new file mode 100644 index 000000000..a76dd975f --- /dev/null +++ b/java/src/test/java/org/forstdb/PutCFVariantsTest.java @@ -0,0 +1,126 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.forstdb.MergeTest.longFromByteArray; +import static org.forstdb.MergeTest.longToByteArray; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class PutCFVariantsTest { + @FunctionalInterface + interface FunctionCFPut { + public void apply(PDatabase db, PColumnFamilyHandle cfh, PLeft two, PRight three) + throws RocksDBException; + } + + @Parameterized.Parameters + public static List> + data() { + return Arrays.asList(RocksDB::put, + (db, cfh, left, right) + -> db.put(cfh, new WriteOptions(), left, right), + (db, cfh, left, right) + -> { + final byte[] left0 = + ("1234567" + new String(left, StandardCharsets.UTF_8) + "890").getBytes(); + final byte[] right0 = + ("1234" + new String(right, StandardCharsets.UTF_8) + "567890ab").getBytes(); + db.put(cfh, left0, 7, left.length, right0, 4, right.length); + }, + (db, cfh, left, right) + -> { + final byte[] left0 = + ("1234567" + new String(left, StandardCharsets.UTF_8) + "890").getBytes(); + final byte[] right0 = + ("1234" + new String(right, StandardCharsets.UTF_8) + "567890ab").getBytes(); + db.put(cfh, new WriteOptions(), left0, 7, left.length, right0, 4, right.length); + }, + + (db, cfh, left, right) + -> { + final ByteBuffer bbLeft = ByteBuffer.allocateDirect(100); + final ByteBuffer bbRight = ByteBuffer.allocateDirect(100); + bbLeft.put(left).flip(); + bbRight.put(right).flip(); + db.put(cfh, new WriteOptions(), bbLeft, bbRight); + }, + (db, cfh, left, right) -> { + final ByteBuffer bbLeft = ByteBuffer.allocate(100); + final ByteBuffer bbRight = ByteBuffer.allocate(100); + bbLeft.put(left).flip(); + bbRight.put(right).flip(); + db.put(cfh, new WriteOptions(), bbLeft, bbRight); + }); + } + + @Parameterized.Parameter + public PutCFVariantsTest.FunctionCFPut putFunction; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void writeAndRead() throws InterruptedException, RocksDBException { + try (final UInt64AddOperator uint64AddOperator = new UInt64AddOperator(); + final ColumnFamilyOptions cfOpt1 = + new ColumnFamilyOptions().setMergeOperator(uint64AddOperator); + final ColumnFamilyOptions cfOpt2 = + new ColumnFamilyOptions().setMergeOperator(uint64AddOperator)) { + final List cfDescriptors = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpt1), + new ColumnFamilyDescriptor("new_cf".getBytes(), cfOpt2)); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions opt = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open( + opt, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList)) { + try { + // writing (long)100 under key + putFunction.apply( + db, columnFamilyHandleList.get(1), "cfkey".getBytes(), longToByteArray(100)); + // merge (long)1 under key + byte[] value = db.get(columnFamilyHandleList.get(1), "cfkey".getBytes()); + final long longValue = longFromByteArray(value); + + // Test also with createColumnFamily + try (final ColumnFamilyOptions cfHandleOpts = + new ColumnFamilyOptions().setMergeOperator(uint64AddOperator); + final ColumnFamilyHandle cfHandle = db.createColumnFamily( + new ColumnFamilyDescriptor("new_cf2".getBytes(), cfHandleOpts))) { + // writing (long)200 under cfkey2 + db.put(cfHandle, "cfkey2".getBytes(), longToByteArray(200)); + // merge (long)50 under cfkey2 + value = db.get(cfHandle, "cfkey2".getBytes()); + final long longValueTmpCf = longFromByteArray(value); + + assertThat(longValue).isEqualTo(100); + assertThat(longValueTmpCf).isEqualTo(200); + } + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + } +} diff --git a/java/src/test/java/org/rocksdb/PutMultiplePartsTest.java b/java/src/test/java/org/forstdb/PutMultiplePartsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/PutMultiplePartsTest.java rename to java/src/test/java/org/forstdb/PutMultiplePartsTest.java index 7835737ae..4846c2537 100644 --- a/java/src/test/java/org/rocksdb/PutMultiplePartsTest.java +++ b/java/src/test/java/org/forstdb/PutMultiplePartsTest.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/forstdb/PutVariantsTest.java b/java/src/test/java/org/forstdb/PutVariantsTest.java new file mode 100644 index 000000000..ce6bbeeda --- /dev/null +++ b/java/src/test/java/org/forstdb/PutVariantsTest.java @@ -0,0 +1,92 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.forstdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.forstdb.MergeTest.longFromByteArray; +import static org.forstdb.MergeTest.longToByteArray; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class PutVariantsTest { + @FunctionalInterface + interface FunctionPut { + public void apply(PDatabase db, PLeft two, PRight three) throws RocksDBException; + } + + @Parameterized.Parameters + public static List> data() { + return Arrays.asList(RocksDB::put, + (db, left, right) + -> db.put(new WriteOptions(), left, right), + (db, left, right) + -> { + final byte[] left0 = + ("1234567" + new String(left, StandardCharsets.UTF_8) + "890").getBytes(); + final byte[] right0 = + ("1234" + new String(right, StandardCharsets.UTF_8) + "567890ab").getBytes(); + db.put(left0, 7, left.length, right0, 4, right.length); + }, + (db, left, right) + -> { + final byte[] left0 = + ("1234567" + new String(left, StandardCharsets.UTF_8) + "890").getBytes(); + final byte[] right0 = + ("1234" + new String(right, StandardCharsets.UTF_8) + "567890ab").getBytes(); + db.put(new WriteOptions(), left0, 7, left.length, right0, 4, right.length); + }, + (db, left, right) + -> { + final ByteBuffer bbLeft = ByteBuffer.allocateDirect(100); + final ByteBuffer bbRight = ByteBuffer.allocateDirect(100); + bbLeft.put(left).flip(); + bbRight.put(right).flip(); + db.put(new WriteOptions(), bbLeft, bbRight); + }, + (db, left, right) -> { + final ByteBuffer bbLeft = ByteBuffer.allocate(100); + final ByteBuffer bbRight = ByteBuffer.allocate(100); + bbLeft.put(left).flip(); + bbRight.put(right).flip(); + db.put(new WriteOptions(), bbLeft, bbRight); + }); + } + + @Parameterized.Parameter public PutVariantsTest.FunctionPut putFunction; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void writeAndRead() throws InterruptedException, RocksDBException { + try (final UInt64AddOperator uint64AddOperator = new UInt64AddOperator(); + final Options opt = + new Options().setCreateIfMissing(true).setMergeOperator(uint64AddOperator); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + // Writing (long)100 under key + putFunction.apply(db, "key".getBytes(), longToByteArray(100)); + + final byte[] value = db.get("key".getBytes()); + final long longValue = longFromByteArray(value); + + assertThat(longValue).isEqualTo(100); + } + } +} diff --git a/java/src/test/java/org/rocksdb/RateLimiterTest.java b/java/src/test/java/org/forstdb/RateLimiterTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/RateLimiterTest.java rename to java/src/test/java/org/forstdb/RateLimiterTest.java index e7d6e6c49..5e834bcc0 100644 --- a/java/src/test/java/org/rocksdb/RateLimiterTest.java +++ b/java/src/test/java/org/forstdb/RateLimiterTest.java @@ -2,13 +2,13 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.rocksdb.RateLimiter.*; +import static org.forstdb.RateLimiter.*; public class RateLimiterTest { diff --git a/java/src/test/java/org/rocksdb/ReadOnlyTest.java b/java/src/test/java/org/forstdb/ReadOnlyTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/ReadOnlyTest.java rename to java/src/test/java/org/forstdb/ReadOnlyTest.java index 99549b61b..573e8307b 100644 --- a/java/src/test/java/org/rocksdb/ReadOnlyTest.java +++ b/java/src/test/java/org/forstdb/ReadOnlyTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/ReadOptionsTest.java b/java/src/test/java/org/forstdb/ReadOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/ReadOptionsTest.java rename to java/src/test/java/org/forstdb/ReadOptionsTest.java index 1bc24b984..f7f6125d9 100644 --- a/java/src/test/java/org/rocksdb/ReadOptionsTest.java +++ b/java/src/test/java/org/forstdb/ReadOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Arrays; import java.util.Random; diff --git a/java/src/test/java/org/rocksdb/RocksDBExceptionTest.java b/java/src/test/java/org/forstdb/RocksDBExceptionTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/RocksDBExceptionTest.java rename to java/src/test/java/org/forstdb/RocksDBExceptionTest.java index d3bd4ece7..8fb9285d2 100644 --- a/java/src/test/java/org/rocksdb/RocksDBExceptionTest.java +++ b/java/src/test/java/org/forstdb/RocksDBExceptionTest.java @@ -3,12 +3,12 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; -import org.rocksdb.Status.Code; -import org.rocksdb.Status.SubCode; +import org.forstdb.Status.Code; +import org.forstdb.Status.SubCode; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; diff --git a/java/src/test/java/org/rocksdb/RocksDBTest.java b/java/src/test/java/org/forstdb/RocksDBTest.java similarity index 85% rename from java/src/test/java/org/rocksdb/RocksDBTest.java rename to java/src/test/java/org/forstdb/RocksDBTest.java index 3da65a848..b5e6a1eb4 100644 --- a/java/src/test/java/org/rocksdb/RocksDBTest.java +++ b/java/src/test/java/org/forstdb/RocksDBTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.*; import org.junit.rules.ExpectedException; @@ -184,10 +184,8 @@ public void put() throws RocksDBException { final WriteOptions opt = new WriteOptions(); final ReadOptions optr = new ReadOptions()) { db.put("key1".getBytes(), "value".getBytes()); db.put(opt, "key2".getBytes(), "12345678".getBytes()); - assertThat(db.get("key1".getBytes())).isEqualTo( - "value".getBytes()); - assertThat(db.get("key2".getBytes())).isEqualTo( - "12345678".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("12345678".getBytes()); final ByteBuffer key = ByteBuffer.allocateDirect(12); final ByteBuffer value = ByteBuffer.allocateDirect(12); @@ -245,8 +243,73 @@ public void put() throws RocksDBException { } } - private static Segment sliceSegment(final String key) { - final ByteBuffer rawKey = ByteBuffer.allocate(key.length() + 4); + @Test + public void putIndirectByteBuffers() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final WriteOptions opt = new WriteOptions(); final ReadOptions optr = new ReadOptions()) { + db.put("key1".getBytes(), "value".getBytes()); + db.put(opt, "key2".getBytes(), "12345678".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("12345678".getBytes()); + + ByteBuffer key = ByteBuffer.allocate(12); + ByteBuffer value = ByteBuffer.allocate(12); + key.position(4); + key.put("key3".getBytes()); + key.position(4).limit(8); + value.position(4); + value.put("val3".getBytes()); + value.position(4).limit(8); + + db.put(opt, key, value); + + assertThat(key.position()).isEqualTo(8); + assertThat(key.limit()).isEqualTo(8); + + assertThat(value.position()).isEqualTo(8); + assertThat(value.limit()).isEqualTo(8); + + key.position(4); + + ByteBuffer result = ByteBuffer.allocate(12); + assertThat(db.get(optr, key, result)).isEqualTo(4); + assertThat(result.position()).isEqualTo(0); + assertThat(result.limit()).isEqualTo(4); + assertThat(key.position()).isEqualTo(8); + assertThat(key.limit()).isEqualTo(8); + + byte[] tmp = new byte[4]; + result.get(tmp); + assertThat(tmp).isEqualTo("val3".getBytes()); + + key.position(4); + + result.clear().position(9); + assertThat(db.get(optr, key, result)).isEqualTo(4); + assertThat(result.position()).isEqualTo(9); + assertThat(result.limit()).isEqualTo(12); + assertThat(key.position()).isEqualTo(8); + assertThat(key.limit()).isEqualTo(8); + byte[] tmp2 = new byte[3]; + result.get(tmp2); + assertThat(tmp2).isEqualTo("val".getBytes()); + + // put + Segment key3 = sliceSegment("key3"); + Segment key4 = sliceSegment("key4"); + Segment value0 = sliceSegment("value 0"); + Segment value1 = sliceSegment("value 1"); + db.put(key3.data, key3.offset, key3.len, value0.data, value0.offset, value0.len); + db.put(opt, key4.data, key4.offset, key4.len, value1.data, value1.offset, value1.len); + + // compare + Assert.assertTrue(value0.isSamePayload(db.get(key3.data, key3.offset, key3.len))); + Assert.assertTrue(value1.isSamePayload(db.get(key4.data, key4.offset, key4.len))); + } + } + + private static Segment sliceSegment(String key) { + ByteBuffer rawKey = ByteBuffer.allocate(key.length() + 4); rawKey.put((byte)0); rawKey.put((byte)0); rawKey.put(key.getBytes()); @@ -546,6 +609,28 @@ public void deleteRange() throws RocksDBException { } } + @Test + public void clipColumnFamily() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + db.put("key3".getBytes(), "abcdefg".getBytes()); + db.put("key4".getBytes(), "xyz".getBytes()); + db.put("key5".getBytes(), "qwer".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("12345678".getBytes()); + assertThat(db.get("key3".getBytes())).isEqualTo("abcdefg".getBytes()); + assertThat(db.get("key4".getBytes())).isEqualTo("xyz".getBytes()); + assertThat(db.get("key5".getBytes())).isEqualTo("qwer".getBytes()); + db.clipColumnFamily(db.getDefaultColumnFamily(), "key2".getBytes(), "key4".getBytes()); + assertThat(db.get("key1".getBytes())).isNull(); + assertThat(db.get("key2".getBytes())).isEqualTo("12345678".getBytes()); + assertThat(db.get("key3".getBytes())).isEqualTo("abcdefg".getBytes()); + assertThat(db.get("key4".getBytes())).isNull(); + assertThat(db.get("key5".getBytes())).isNull(); + } + } + @Test public void getIntProperty() throws RocksDBException { try ( @@ -1051,6 +1136,40 @@ public void compactRangeToLevelColumnFamily() } } + @Test + public void compactRangeWithNullBoundaries() throws RocksDBException { + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setDisableAutoCompactions(true) + .setCompactionStyle(CompactionStyle.LEVEL) + .setNumLevels(4) + .setWriteBufferSize(100 << 10) + .setLevelZeroFileNumCompactionTrigger(3) + .setTargetFileSizeBase(200 << 10) + .setTargetFileSizeMultiplier(1) + .setMaxBytesForLevelBase(500 << 10) + .setMaxBytesForLevelMultiplier(1) + .setDisableAutoCompactions(true); + final FlushOptions flushOptions = new FlushOptions(); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + final byte[] b = new byte[10000]; + // Create an SST containing key4, key5, and key6 + db.put(("key4").getBytes(), b); + db.put(("key5").getBytes(), b); + db.put(("key6").getBytes(), b); + db.flush(flushOptions); + // Create a new SST that includes the tombstones of all keys + db.delete(("key4").getBytes()); + db.delete(("key5").getBytes()); + db.delete(("key6").getBytes()); + db.flush(flushOptions); + + db.compactRange(("key4").getBytes(), null); + List liveFilesMetaData = db.getLiveFilesMetaData(); + assertThat(liveFilesMetaData.size()).isEqualTo(0); + } + } + @Test public void continueBackgroundWorkAfterCancelAllBackgroundWork() throws RocksDBException { final int KEY_SIZE = 20; @@ -1262,6 +1381,61 @@ public void getApproximateMemTableStatsSingleKey() throws RocksDBException { } } + @Test + public void getLiveFilesMetadataWithChecksum() throws RocksDBException { + final Properties props = new Properties(); + final byte[] key1 = "key1".getBytes(UTF_8); + props.put("file_checksum_gen_factory", "FileChecksumGenCrc32cFactory"); + + try (final DBOptions dbOptions = DBOptions.getDBOptionsFromProps(props); + final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); + final Options options = new Options(dbOptions, cfOptions).setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + + // disable WAL so we have a deterministic checksum + try (final RocksDB db = RocksDB.open(options, dbPath); + final WriteOptions writeOptions = new WriteOptions().setDisableWAL(true)) { + db.put(writeOptions, key1, key1); + } + + try (final RocksDB db = RocksDB.open(options, dbPath)) { + final List expectedFileMetadata = db.getLiveFilesMetaData(); + assertThat(expectedFileMetadata).hasSize(1); + // ideally we could re-compute here, but CRC32C is a Java 9 feature, so we have no CRC32C + // implementation available here + final LiveFileMetaData sstFile = expectedFileMetadata.get(0); + assertThat(sstFile.fileChecksum()).isNotEmpty(); + } + } + } + + @Test + public void getColumnFamilyMetadataWithChecksum() throws RocksDBException { + final Properties props = new Properties(); + props.put("file_checksum_gen_factory", "FileChecksumGenCrc32cFactory"); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + + try (final DBOptions dbOptions = DBOptions.getDBOptionsFromProps(props); + final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); + final Options options = new Options(dbOptions, cfOptions).setCreateIfMissing(true)) { + try (final RocksDB db = RocksDB.open(options, dbPath); + final WriteOptions writeOptions = new WriteOptions().setDisableWAL(true)) { + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + } + + try (final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + ColumnFamilyMetaData metadata = db.getColumnFamilyMetaData(); // Exception here + List levels = metadata.levels(); + assertThat(levels).isNotEmpty(); + List filesMetadata = levels.get(0).files(); + assertThat(filesMetadata).isNotEmpty(); + assertThat(filesMetadata.get(0).fileChecksum()).isNotNull(); + assertThat(filesMetadata.get(0).fileChecksum()).hasSize(4); + assertThat(filesMetadata.get(0).fileChecksum()).isNotEqualTo(new byte[] {0, 0, 0, 0}); + } + } + } + @Ignore("TODO(AR) re-enable when ready!") @Test public void compactFiles() throws RocksDBException { @@ -1346,6 +1520,25 @@ public void enableAutoCompaction() throws RocksDBException { } } + @Test + public void enableAutoCompactionNull() throws RocksDBException { + try (final DBOptions options = new DBOptions().setCreateIfMissing(true)) { + final List cfDescs = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List cfHandles = new ArrayList<>(); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath, cfDescs, cfHandles)) { + try { + db.enableAutoCompaction(null); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } + @Test public void numberLevels() throws RocksDBException { try (final Options options = new Options().setCreateIfMissing(true)) { @@ -1434,7 +1627,7 @@ public void getLiveFiles() throws RocksDBException { try (final RocksDB db = RocksDB.open(options, dbPath)) { final RocksDB.LiveFiles livefiles = db.getLiveFiles(true); assertThat(livefiles).isNotNull(); - assertThat(livefiles.manifestFileSize).isEqualTo(66); + assertThat(livefiles.manifestFileSize).isEqualTo(70); assertThat(livefiles.files.size()).isEqualTo(3); assertThat(livefiles.files.get(0)).isEqualTo("/CURRENT"); assertThat(livefiles.files.get(1)).isEqualTo("/MANIFEST-000005"); @@ -1579,9 +1772,43 @@ public void suggestCompactRange() throws RocksDBException { db.put(cfHandles.get(0), "key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); db.put(cfHandles.get(0), "key2".getBytes(UTF_8), "value2".getBytes(UTF_8)); db.put(cfHandles.get(0), "key3".getBytes(UTF_8), "value3".getBytes(UTF_8)); + try { + final Range range = db.suggestCompactRange(); + assertThat(range).isNotNull(); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } + + @Test + public void suggestCompactRangeCF() throws RocksDBException { + try (final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true)) { + final List cfDescs = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new ColumnFamilyOptions()), + new ColumnFamilyDescriptor("new_cf2".getBytes(), new ColumnFamilyOptions())); + + final List cfHandles = new ArrayList<>(); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath, cfDescs, cfHandles)) { + db.put(cfHandles.get(0), "key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + db.put(cfHandles.get(0), "key2".getBytes(UTF_8), "value2".getBytes(UTF_8)); + db.put(cfHandles.get(0), "key3".getBytes(UTF_8), "value3".getBytes(UTF_8)); + db.put(cfHandles.get(1), "key1_new_cf".getBytes(UTF_8), "value1".getBytes(UTF_8)); + db.put(cfHandles.get(1), "key2_new_cf".getBytes(UTF_8), "value2".getBytes(UTF_8)); + db.put(cfHandles.get(1), "key3_new_cf".getBytes(UTF_8), "value3".getBytes(UTF_8)); try { final Range range = db.suggestCompactRange(cfHandles.get(0)); assertThat(range).isNotNull(); + final Range rangeCF = db.suggestCompactRange(cfHandles.get(1)); + assertThat(rangeCF).isNotNull(); + final Range rangeCFEmpty = db.suggestCompactRange(cfHandles.get(2)); + assertThat(rangeCFEmpty).isNotNull(); } finally { for (final ColumnFamilyHandle cfHandle : cfHandles) { cfHandle.close(); diff --git a/java/src/test/java/org/rocksdb/RocksIteratorTest.java b/java/src/test/java/org/forstdb/RocksIteratorTest.java similarity index 69% rename from java/src/test/java/org/rocksdb/RocksIteratorTest.java rename to java/src/test/java/org/forstdb/RocksIteratorTest.java index 2a13550b7..34eb5e779 100644 --- a/java/src/test/java/org/rocksdb/RocksIteratorTest.java +++ b/java/src/test/java/org/forstdb/RocksIteratorTest.java @@ -2,9 +2,10 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -45,7 +46,7 @@ private void validateValue( } @Test - public void rocksIterator() throws RocksDBException { + public void rocksIteratorByteBuffers() throws RocksDBException { try (final Options options = new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { @@ -72,6 +73,103 @@ public void rocksIterator() throws RocksDBException { validateKey(iterator, ByteBuffer.allocate(5), "key1"); validateValue(iterator, ByteBuffer.allocate(2), "value1"); validateValue(iterator, ByteBuffer.allocate(8), "value1"); + } + } + } + + @Test + public void rocksIteratorByteArrayValues() throws RocksDBException { + try (final Options options = + new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1".getBytes()); + db.put("key2".getBytes(), "value2".getBytes()); + + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + + final byte[] valueArray0 = new byte[2]; + assertThat(iterator.value(valueArray0)).isEqualTo(6); + assertThat(valueArray0).isEqualTo("va".getBytes()); + final byte[] valueArray1 = new byte[8]; + assertThat(iterator.value(valueArray1)).isEqualTo(6); + assertThat(valueArray1).isEqualTo("value1\0\0".getBytes()); + final byte[] valueArray2 = new byte[10]; + assertThat(iterator.value(valueArray2, 2, 6)).isEqualTo(6); + assertThat(valueArray2).isEqualTo("\0\0value1\0\0".getBytes()); + final byte[] valueArray3 = new byte[10]; + assertThat(iterator.value(valueArray3, 5, 5)).isEqualTo(6); + assertThat(valueArray3).isEqualTo("\0\0\0\0\0value".getBytes()); + final byte[] valueArray4 = new byte[6]; + try { + iterator.value(valueArray4, 1, 6); + fail("Expected IndexOutOfBoundsException"); + } catch (final IndexOutOfBoundsException ignored) { + // we should arrive here + } + final byte[] valueArray5 = new byte[7]; + assertThat(iterator.value(valueArray5, 1, 6)).isEqualTo(6); + assertThat(valueArray5).isEqualTo("\0value1".getBytes()); + } + } + } + + @Test + public void rocksIteratorByteArrayKeys() throws RocksDBException { + try (final Options options = + new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1".getBytes()); + db.put("key2".getBytes(), "value2".getBytes()); + + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + + final byte[] keyArray0 = new byte[2]; + assertThat(iterator.key(keyArray0)).isEqualTo(4); + assertThat(keyArray0).isEqualTo("ke".getBytes()); + final byte[] keyArray1 = new byte[8]; + assertThat(iterator.key(keyArray1)).isEqualTo(4); + assertThat(keyArray1).isEqualTo("key1\0\0\0\0".getBytes()); + final byte[] keyArray2 = new byte[10]; + assertThat(iterator.key(keyArray2, 2, 6)).isEqualTo(4); + assertThat(keyArray2).isEqualTo("\0\0key1\0\0\0\0".getBytes()); + final byte[] keyArray3 = new byte[10]; + assertThat(iterator.key(keyArray3, 5, 3)).isEqualTo(4); + assertThat(keyArray3).isEqualTo("\0\0\0\0\0key\0\0".getBytes()); + final byte[] keyArray4 = new byte[4]; + try { + iterator.key(keyArray4, 1, 4); + fail("Expected IndexOutOfBoundsException"); + } catch (final IndexOutOfBoundsException ignored) { + // we should arrive here + } + final byte[] keyArray5 = new byte[5]; + assertThat(iterator.key(keyArray5, 1, 4)).isEqualTo(4); + assertThat(keyArray5).isEqualTo("\0key1".getBytes()); + } + } + } + + @Test + public void rocksIteratorSimple() throws RocksDBException { + try (final Options options = + new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1".getBytes()); + db.put("key2".getBytes(), "value2".getBytes()); + + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); iterator.next(); assertThat(iterator.isValid()).isTrue(); @@ -90,6 +188,23 @@ public void rocksIterator() throws RocksDBException { assertThat(iterator.key()).isEqualTo("key2".getBytes()); assertThat(iterator.value()).isEqualTo("value2".getBytes()); iterator.status(); + } + } + } + + @Test + public void rocksIterator() throws RocksDBException { + try (final Options options = + new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1".getBytes()); + db.put("key2".getBytes(), "value2".getBytes()); + + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); { final ByteBuffer key = ByteBuffer.allocate(12); diff --git a/java/src/test/java/org/rocksdb/RocksMemEnvTest.java b/java/src/test/java/org/forstdb/RocksMemEnvTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/RocksMemEnvTest.java rename to java/src/test/java/org/forstdb/RocksMemEnvTest.java index 40b24ffa3..5f77ce3e3 100644 --- a/java/src/test/java/org/rocksdb/RocksMemEnvTest.java +++ b/java/src/test/java/org/forstdb/RocksMemEnvTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/RocksNativeLibraryResource.java b/java/src/test/java/org/forstdb/RocksNativeLibraryResource.java similarity index 95% rename from java/src/test/java/org/rocksdb/RocksNativeLibraryResource.java rename to java/src/test/java/org/forstdb/RocksNativeLibraryResource.java index 6116f2f92..7f85047ac 100644 --- a/java/src/test/java/org/rocksdb/RocksNativeLibraryResource.java +++ b/java/src/test/java/org/forstdb/RocksNativeLibraryResource.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.rules.ExternalResource; diff --git a/java/src/test/java/org/rocksdb/SecondaryDBTest.java b/java/src/test/java/org/forstdb/SecondaryDBTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/SecondaryDBTest.java rename to java/src/test/java/org/forstdb/SecondaryDBTest.java index 557d4a47d..99a76afbd 100644 --- a/java/src/test/java/org/rocksdb/SecondaryDBTest.java +++ b/java/src/test/java/org/forstdb/SecondaryDBTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/SliceTest.java b/java/src/test/java/org/forstdb/SliceTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/SliceTest.java rename to java/src/test/java/org/forstdb/SliceTest.java index c65b01903..ec83cd419 100644 --- a/java/src/test/java/org/rocksdb/SliceTest.java +++ b/java/src/test/java/org/forstdb/SliceTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/SnapshotTest.java b/java/src/test/java/org/forstdb/SnapshotTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/SnapshotTest.java rename to java/src/test/java/org/forstdb/SnapshotTest.java index 11f0d560a..b6f37ac55 100644 --- a/java/src/test/java/org/rocksdb/SnapshotTest.java +++ b/java/src/test/java/org/forstdb/SnapshotTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/rocksdb/SstFileManagerTest.java b/java/src/test/java/org/forstdb/SstFileManagerTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/SstFileManagerTest.java rename to java/src/test/java/org/forstdb/SstFileManagerTest.java index 2e136e820..e1976134e 100644 --- a/java/src/test/java/org/rocksdb/SstFileManagerTest.java +++ b/java/src/test/java/org/forstdb/SstFileManagerTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/SstFileReaderTest.java b/java/src/test/java/org/forstdb/SstFileReaderTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/SstFileReaderTest.java rename to java/src/test/java/org/forstdb/SstFileReaderTest.java index ef74b08a7..959558ffe 100644 --- a/java/src/test/java/org/rocksdb/SstFileReaderTest.java +++ b/java/src/test/java/org/forstdb/SstFileReaderTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; @@ -20,7 +20,7 @@ import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.rocksdb.util.ByteBufferAllocator; +import org.forstdb.util.ByteBufferAllocator; @RunWith(Parameterized.class) public class SstFileReaderTest { diff --git a/java/src/test/java/org/rocksdb/SstFileWriterTest.java b/java/src/test/java/org/forstdb/SstFileWriterTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/SstFileWriterTest.java rename to java/src/test/java/org/forstdb/SstFileWriterTest.java index c0f4ed9f1..7e686eb35 100644 --- a/java/src/test/java/org/rocksdb/SstFileWriterTest.java +++ b/java/src/test/java/org/forstdb/SstFileWriterTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -18,7 +18,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.BytewiseComparator; +import org.forstdb.util.BytewiseComparator; public class SstFileWriterTest { private static final String SST_FILE_NAME = "test.sst"; diff --git a/java/src/test/java/org/rocksdb/SstPartitionerTest.java b/java/src/test/java/org/forstdb/SstPartitionerTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/SstPartitionerTest.java rename to java/src/test/java/org/forstdb/SstPartitionerTest.java index 3ee739053..48b225d32 100644 --- a/java/src/test/java/org/rocksdb/SstPartitionerTest.java +++ b/java/src/test/java/org/forstdb/SstPartitionerTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/StatisticsCollectorTest.java b/java/src/test/java/org/forstdb/StatisticsCollectorTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/StatisticsCollectorTest.java rename to java/src/test/java/org/forstdb/StatisticsCollectorTest.java index 36721c80d..20436b687 100644 --- a/java/src/test/java/org/rocksdb/StatisticsCollectorTest.java +++ b/java/src/test/java/org/forstdb/StatisticsCollectorTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Collections; diff --git a/java/src/test/java/org/rocksdb/StatisticsTest.java b/java/src/test/java/org/forstdb/StatisticsTest.java similarity index 92% rename from java/src/test/java/org/rocksdb/StatisticsTest.java rename to java/src/test/java/org/forstdb/StatisticsTest.java index de92102ec..3e83fae2c 100644 --- a/java/src/test/java/org/rocksdb/StatisticsTest.java +++ b/java/src/test/java/org/forstdb/StatisticsTest.java @@ -3,26 +3,30 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.util.EnumSet; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import java.nio.charset.StandardCharsets; - -import static org.assertj.core.api.Assertions.assertThat; - public class StatisticsTest { - - @ClassRule - public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = - new RocksNativeLibraryResource(); - @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + @Test + public void createStatistics() throws RocksDBException { + final Statistics statistics = new Statistics(); + statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS); + final Statistics statisticsWithHistogramOptions = + new Statistics(EnumSet.of(HistogramType.DB_WRITE, HistogramType.COMPACTION_TIME)); + statisticsWithHistogramOptions.reset(); + } + @Test public void statsLevel() throws RocksDBException { final Statistics statistics = new Statistics(); diff --git a/java/src/test/java/org/rocksdb/StatsCallbackMock.java b/java/src/test/java/org/forstdb/StatsCallbackMock.java similarity index 96% rename from java/src/test/java/org/rocksdb/StatsCallbackMock.java rename to java/src/test/java/org/forstdb/StatsCallbackMock.java index c6a7294c9..24cafd018 100644 --- a/java/src/test/java/org/rocksdb/StatsCallbackMock.java +++ b/java/src/test/java/org/forstdb/StatsCallbackMock.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; public class StatsCallbackMock implements StatisticsCollectorCallback { public int tickerCallbackCount = 0; diff --git a/java/src/test/java/org/rocksdb/TableFilterTest.java b/java/src/test/java/org/forstdb/TableFilterTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/TableFilterTest.java rename to java/src/test/java/org/forstdb/TableFilterTest.java index 2bd3b1798..c9604f823 100644 --- a/java/src/test/java/org/rocksdb/TableFilterTest.java +++ b/java/src/test/java/org/forstdb/TableFilterTest.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import org.junit.Rule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/TimedEnvTest.java b/java/src/test/java/org/forstdb/TimedEnvTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/TimedEnvTest.java rename to java/src/test/java/org/forstdb/TimedEnvTest.java index 31bad2e2e..3134a131a 100644 --- a/java/src/test/java/org/rocksdb/TimedEnvTest.java +++ b/java/src/test/java/org/forstdb/TimedEnvTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/rocksdb/TransactionDBOptionsTest.java b/java/src/test/java/org/forstdb/TransactionDBOptionsTest.java similarity index 98% rename from java/src/test/java/org/rocksdb/TransactionDBOptionsTest.java rename to java/src/test/java/org/forstdb/TransactionDBOptionsTest.java index 7eaa6b16c..303da19ec 100644 --- a/java/src/test/java/org/rocksdb/TransactionDBOptionsTest.java +++ b/java/src/test/java/org/forstdb/TransactionDBOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/TransactionDBTest.java b/java/src/test/java/org/forstdb/TransactionDBTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/TransactionDBTest.java rename to java/src/test/java/org/forstdb/TransactionDBTest.java index 56acb21c7..4e0e3848b 100644 --- a/java/src/test/java/org/rocksdb/TransactionDBTest.java +++ b/java/src/test/java/org/forstdb/TransactionDBTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Rule; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/TransactionLogIteratorTest.java b/java/src/test/java/org/forstdb/TransactionLogIteratorTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/TransactionLogIteratorTest.java rename to java/src/test/java/org/forstdb/TransactionLogIteratorTest.java index 3c4dff7bb..fd96c34ed 100644 --- a/java/src/test/java/org/rocksdb/TransactionLogIteratorTest.java +++ b/java/src/test/java/org/forstdb/TransactionLogIteratorTest.java @@ -1,5 +1,5 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/rocksdb/TransactionOptionsTest.java b/java/src/test/java/org/forstdb/TransactionOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/TransactionOptionsTest.java rename to java/src/test/java/org/forstdb/TransactionOptionsTest.java index add0439e0..1b8ccae54 100644 --- a/java/src/test/java/org/rocksdb/TransactionOptionsTest.java +++ b/java/src/test/java/org/forstdb/TransactionOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/TransactionTest.java b/java/src/test/java/org/forstdb/TransactionTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/TransactionTest.java rename to java/src/test/java/org/forstdb/TransactionTest.java index b80445c5c..9f239788a 100644 --- a/java/src/test/java/org/rocksdb/TransactionTest.java +++ b/java/src/test/java/org/forstdb/TransactionTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; @@ -416,12 +416,13 @@ public TransactionDBContainer startDb() throws RocksDBException { .setCreateIfMissing(true) .setCreateMissingColumnFamilies(true); final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final ColumnFamilyOptions defaultColumnFamilyOptions = new ColumnFamilyOptions(); + defaultColumnFamilyOptions.setMergeOperator(new StringAppendOperator("++")); final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); - final List columnFamilyDescriptors = - Arrays.asList( - new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), - new ColumnFamilyDescriptor(TXN_TEST_COLUMN_FAMILY, - columnFamilyOptions)); + columnFamilyOptions.setMergeOperator(new StringAppendOperator("**")); + final List columnFamilyDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultColumnFamilyOptions), + new ColumnFamilyDescriptor(TXN_TEST_COLUMN_FAMILY, columnFamilyOptions)); final List columnFamilyHandles = new ArrayList<>(); final TransactionDB txnDb; diff --git a/java/src/test/java/org/rocksdb/TtlDBTest.java b/java/src/test/java/org/forstdb/TtlDBTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/TtlDBTest.java rename to java/src/test/java/org/forstdb/TtlDBTest.java index ebf9e9eaa..6457a5e62 100644 --- a/java/src/test/java/org/rocksdb/TtlDBTest.java +++ b/java/src/test/java/org/forstdb/TtlDBTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; diff --git a/java/src/test/java/org/rocksdb/Types.java b/java/src/test/java/org/forstdb/Types.java similarity index 97% rename from java/src/test/java/org/rocksdb/Types.java rename to java/src/test/java/org/forstdb/Types.java index a6abdecbc..d9a0171c5 100644 --- a/java/src/test/java/org/rocksdb/Types.java +++ b/java/src/test/java/org/forstdb/Types.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; /** * Simple type conversion methods diff --git a/java/src/test/java/org/rocksdb/VerifyChecksumsTest.java b/java/src/test/java/org/forstdb/VerifyChecksumsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/VerifyChecksumsTest.java rename to java/src/test/java/org/forstdb/VerifyChecksumsTest.java index ddc2a456f..fbdffca29 100644 --- a/java/src/test/java/org/rocksdb/VerifyChecksumsTest.java +++ b/java/src/test/java/org/forstdb/VerifyChecksumsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/WALRecoveryModeTest.java b/java/src/test/java/org/forstdb/WALRecoveryModeTest.java similarity index 96% rename from java/src/test/java/org/rocksdb/WALRecoveryModeTest.java rename to java/src/test/java/org/forstdb/WALRecoveryModeTest.java index 2a0133f6b..08ee946a9 100644 --- a/java/src/test/java/org/rocksdb/WALRecoveryModeTest.java +++ b/java/src/test/java/org/forstdb/WALRecoveryModeTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/WalFilterTest.java b/java/src/test/java/org/forstdb/WalFilterTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/WalFilterTest.java rename to java/src/test/java/org/forstdb/WalFilterTest.java index 08bc6eef5..5792b60f8 100644 --- a/java/src/test/java/org/rocksdb/WalFilterTest.java +++ b/java/src/test/java/org/forstdb/WalFilterTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.ClassRule; import org.junit.Rule; @@ -16,8 +16,8 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -import static org.rocksdb.util.ByteUtil.bytes; -import static org.rocksdb.util.TestUtil.*; +import static org.forstdb.util.ByteUtil.bytes; +import static org.forstdb.util.TestUtil.*; public class WalFilterTest { diff --git a/java/src/test/java/org/rocksdb/WriteBatchHandlerTest.java b/java/src/test/java/org/forstdb/WriteBatchHandlerTest.java similarity index 91% rename from java/src/test/java/org/rocksdb/WriteBatchHandlerTest.java rename to java/src/test/java/org/forstdb/WriteBatchHandlerTest.java index 2826b128f..6b101f980 100644 --- a/java/src/test/java/org/rocksdb/WriteBatchHandlerTest.java +++ b/java/src/test/java/org/forstdb/WriteBatchHandlerTest.java @@ -3,18 +3,18 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import java.util.Arrays; import java.util.List; import org.junit.ClassRule; import org.junit.Test; -import org.rocksdb.util.CapturingWriteBatchHandler; -import org.rocksdb.util.CapturingWriteBatchHandler.Event; +import org.forstdb.util.CapturingWriteBatchHandler; +import org.forstdb.util.CapturingWriteBatchHandler.Event; import static org.assertj.core.api.Assertions.assertThat; -import static org.rocksdb.util.CapturingWriteBatchHandler.Action.*; +import static org.forstdb.util.CapturingWriteBatchHandler.Action.*; public class WriteBatchHandlerTest { diff --git a/java/src/test/java/org/rocksdb/WriteBatchTest.java b/java/src/test/java/org/forstdb/WriteBatchTest.java similarity index 96% rename from java/src/test/java/org/rocksdb/WriteBatchTest.java rename to java/src/test/java/org/forstdb/WriteBatchTest.java index cc3ad26eb..d36944d05 100644 --- a/java/src/test/java/org/rocksdb/WriteBatchTest.java +++ b/java/src/test/java/org/forstdb/WriteBatchTest.java @@ -6,16 +6,16 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; -import static org.rocksdb.util.CapturingWriteBatchHandler.Action.DELETE; -import static org.rocksdb.util.CapturingWriteBatchHandler.Action.DELETE_RANGE; -import static org.rocksdb.util.CapturingWriteBatchHandler.Action.LOG; -import static org.rocksdb.util.CapturingWriteBatchHandler.Action.MERGE; -import static org.rocksdb.util.CapturingWriteBatchHandler.Action.PUT; -import static org.rocksdb.util.CapturingWriteBatchHandler.Action.SINGLE_DELETE; +import static org.forstdb.util.CapturingWriteBatchHandler.Action.DELETE; +import static org.forstdb.util.CapturingWriteBatchHandler.Action.DELETE_RANGE; +import static org.forstdb.util.CapturingWriteBatchHandler.Action.LOG; +import static org.forstdb.util.CapturingWriteBatchHandler.Action.MERGE; +import static org.forstdb.util.CapturingWriteBatchHandler.Action.PUT; +import static org.forstdb.util.CapturingWriteBatchHandler.Action.SINGLE_DELETE; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; @@ -23,9 +23,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.CapturingWriteBatchHandler; -import org.rocksdb.util.CapturingWriteBatchHandler.Event; -import org.rocksdb.util.WriteBatchGetter; +import org.forstdb.util.CapturingWriteBatchHandler; +import org.forstdb.util.CapturingWriteBatchHandler.Event; +import org.forstdb.util.WriteBatchGetter; /** * This class mimics the db/write_batch_test.cc diff --git a/java/src/test/java/org/rocksdb/WriteBatchThreadedTest.java b/java/src/test/java/org/forstdb/WriteBatchThreadedTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/WriteBatchThreadedTest.java rename to java/src/test/java/org/forstdb/WriteBatchThreadedTest.java index 0321da3fa..ce76c69ae 100644 --- a/java/src/test/java/org/rocksdb/WriteBatchThreadedTest.java +++ b/java/src/test/java/org/forstdb/WriteBatchThreadedTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import org.junit.After; import org.junit.Before; diff --git a/java/src/test/java/org/rocksdb/WriteBatchWithIndexTest.java b/java/src/test/java/org/forstdb/WriteBatchWithIndexTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/WriteBatchWithIndexTest.java rename to java/src/test/java/org/forstdb/WriteBatchWithIndexTest.java index b0a0cdc0e..3a6ccf0e9 100644 --- a/java/src/test/java/org/rocksdb/WriteBatchWithIndexTest.java +++ b/java/src/test/java/org/forstdb/WriteBatchWithIndexTest.java @@ -7,7 +7,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -package org.rocksdb; +package org.forstdb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; @@ -20,7 +20,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.util.ByteBufferAllocator; +import org.forstdb.util.ByteBufferAllocator; public class WriteBatchWithIndexTest { diff --git a/java/src/test/java/org/rocksdb/WriteOptionsTest.java b/java/src/test/java/org/forstdb/WriteOptionsTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/WriteOptionsTest.java rename to java/src/test/java/org/forstdb/WriteOptionsTest.java index 1e1c93fb5..b4092e17a 100644 --- a/java/src/test/java/org/rocksdb/WriteOptionsTest.java +++ b/java/src/test/java/org/forstdb/WriteOptionsTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb; +package org.forstdb; import static org.assertj.core.api.Assertions.assertThat; diff --git a/java/src/test/java/org/rocksdb/flink/FlinkEnvTest.java b/java/src/test/java/org/forstdb/flink/FlinkEnvTest.java similarity index 92% rename from java/src/test/java/org/rocksdb/flink/FlinkEnvTest.java rename to java/src/test/java/org/forstdb/flink/FlinkEnvTest.java index 5c7166557..805755637 100644 --- a/java/src/test/java/org/rocksdb/flink/FlinkEnvTest.java +++ b/java/src/test/java/org/forstdb/flink/FlinkEnvTest.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package org.rocksdb.flink; +package org.forstdb.flink; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.EnvFlinkTestSuite; -import org.rocksdb.RocksNativeLibraryResource; +import org.forstdb.EnvFlinkTestSuite; +import org.forstdb.RocksNativeLibraryResource; /** * Unit test for env/flink/env_flink.cc. diff --git a/java/src/test/java/org/rocksdb/test/RemoveEmptyValueCompactionFilterFactory.java b/java/src/test/java/org/forstdb/test/RemoveEmptyValueCompactionFilterFactory.java similarity index 77% rename from java/src/test/java/org/rocksdb/test/RemoveEmptyValueCompactionFilterFactory.java rename to java/src/test/java/org/forstdb/test/RemoveEmptyValueCompactionFilterFactory.java index c4e4f25a0..b93346f8a 100644 --- a/java/src/test/java/org/rocksdb/test/RemoveEmptyValueCompactionFilterFactory.java +++ b/java/src/test/java/org/forstdb/test/RemoveEmptyValueCompactionFilterFactory.java @@ -1,9 +1,9 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb.test; +package org.forstdb.test; -import org.rocksdb.AbstractCompactionFilter; -import org.rocksdb.AbstractCompactionFilterFactory; -import org.rocksdb.RemoveEmptyValueCompactionFilter; +import org.forstdb.AbstractCompactionFilter; +import org.forstdb.AbstractCompactionFilterFactory; +import org.forstdb.RemoveEmptyValueCompactionFilter; /** * Simple CompactionFilterFactory class used in tests. Generates RemoveEmptyValueCompactionFilters. diff --git a/java/src/test/java/org/rocksdb/test/RocksJunitRunner.java b/java/src/test/java/org/forstdb/test/RocksJunitRunner.java similarity index 97% rename from java/src/test/java/org/rocksdb/test/RocksJunitRunner.java rename to java/src/test/java/org/forstdb/test/RocksJunitRunner.java index 42d3148ef..81f9cb3a6 100644 --- a/java/src/test/java/org/rocksdb/test/RocksJunitRunner.java +++ b/java/src/test/java/org/forstdb/test/RocksJunitRunner.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.test; +package org.forstdb.test; import org.junit.internal.JUnitSystem; import org.junit.internal.RealSystem; @@ -11,7 +11,7 @@ import org.junit.runner.JUnitCore; import org.junit.runner.Result; import org.junit.runner.notification.Failure; -import org.rocksdb.RocksDB; +import org.forstdb.RocksDB; import java.io.PrintStream; import java.text.DecimalFormat; @@ -19,7 +19,7 @@ import java.util.ArrayList; import java.util.List; -import static org.rocksdb.test.RocksJunitRunner.RocksJunitListener.Status.*; +import static org.forstdb.test.RocksJunitRunner.RocksJunitListener.Status.*; /** * Custom Junit Runner to print also Test classes diff --git a/java/src/test/java/org/rocksdb/test/TestableEventListener.java b/java/src/test/java/org/forstdb/test/TestableEventListener.java similarity index 90% rename from java/src/test/java/org/rocksdb/test/TestableEventListener.java rename to java/src/test/java/org/forstdb/test/TestableEventListener.java index 865ad5cf7..8aecc4688 100644 --- a/java/src/test/java/org/rocksdb/test/TestableEventListener.java +++ b/java/src/test/java/org/forstdb/test/TestableEventListener.java @@ -2,9 +2,9 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.test; +package org.forstdb.test; -import org.rocksdb.AbstractEventListener; +import org.forstdb.AbstractEventListener; public class TestableEventListener extends AbstractEventListener { public TestableEventListener() { diff --git a/java/src/test/java/org/rocksdb/util/ByteBufferAllocator.java b/java/src/test/java/org/forstdb/util/ByteBufferAllocator.java similarity index 94% rename from java/src/test/java/org/rocksdb/util/ByteBufferAllocator.java rename to java/src/test/java/org/forstdb/util/ByteBufferAllocator.java index 8d7956cf2..d8967a4fe 100644 --- a/java/src/test/java/org/rocksdb/util/ByteBufferAllocator.java +++ b/java/src/test/java/org/forstdb/util/ByteBufferAllocator.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import java.nio.ByteBuffer; diff --git a/java/src/test/java/org/rocksdb/util/BytewiseComparatorIntTest.java b/java/src/test/java/org/forstdb/util/BytewiseComparatorIntTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/util/BytewiseComparatorIntTest.java rename to java/src/test/java/org/forstdb/util/BytewiseComparatorIntTest.java index fb7239c92..a5bcd1ff4 100644 --- a/java/src/test/java/org/rocksdb/util/BytewiseComparatorIntTest.java +++ b/java/src/test/java/org/forstdb/util/BytewiseComparatorIntTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -14,7 +14,7 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; -import org.rocksdb.*; +import org.forstdb.*; import java.nio.ByteBuffer; import java.nio.file.FileSystems; diff --git a/java/src/test/java/org/rocksdb/util/BytewiseComparatorTest.java b/java/src/test/java/org/forstdb/util/BytewiseComparatorTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/util/BytewiseComparatorTest.java rename to java/src/test/java/org/forstdb/util/BytewiseComparatorTest.java index 69f2c282b..6f84e1bcc 100644 --- a/java/src/test/java/org/rocksdb/util/BytewiseComparatorTest.java +++ b/java/src/test/java/org/forstdb/util/BytewiseComparatorTest.java @@ -3,13 +3,13 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.rocksdb.*; +import org.forstdb.*; import java.io.IOException; import java.nio.ByteBuffer; @@ -18,7 +18,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.*; -import static org.rocksdb.util.ByteUtil.bytes; +import static org.forstdb.util.ByteUtil.bytes; /** * This is a direct port of various C++ diff --git a/java/src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java b/java/src/test/java/org/forstdb/util/CapturingWriteBatchHandler.java similarity index 98% rename from java/src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java rename to java/src/test/java/org/forstdb/util/CapturingWriteBatchHandler.java index 8ea104332..29f39ab88 100644 --- a/java/src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java +++ b/java/src/test/java/org/forstdb/util/CapturingWriteBatchHandler.java @@ -1,8 +1,8 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb.util; +package org.forstdb.util; -import org.rocksdb.RocksDBException; -import org.rocksdb.WriteBatch; +import org.forstdb.RocksDBException; +import org.forstdb.WriteBatch; import java.util.ArrayList; import java.util.Arrays; diff --git a/java/src/test/java/org/rocksdb/util/DirectByteBufferAllocator.java b/java/src/test/java/org/forstdb/util/DirectByteBufferAllocator.java similarity index 95% rename from java/src/test/java/org/rocksdb/util/DirectByteBufferAllocator.java rename to java/src/test/java/org/forstdb/util/DirectByteBufferAllocator.java index d26fb578b..b5ac81b26 100644 --- a/java/src/test/java/org/rocksdb/util/DirectByteBufferAllocator.java +++ b/java/src/test/java/org/forstdb/util/DirectByteBufferAllocator.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import java.nio.ByteBuffer; diff --git a/java/src/test/java/org/rocksdb/util/EnvironmentTest.java b/java/src/test/java/org/forstdb/util/EnvironmentTest.java similarity index 96% rename from java/src/test/java/org/rocksdb/util/EnvironmentTest.java rename to java/src/test/java/org/forstdb/util/EnvironmentTest.java index ae340e06d..dfb150d69 100644 --- a/java/src/test/java/org/rocksdb/util/EnvironmentTest.java +++ b/java/src/test/java/org/forstdb/util/EnvironmentTest.java @@ -2,10 +2,9 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import static org.assertj.core.api.Assertions.assertThat; -import static org.hamcrest.Matchers.is; import java.lang.reflect.Field; import org.junit.AfterClass; @@ -13,11 +12,11 @@ import org.junit.Test; public class EnvironmentTest { - private final static String ARCH_FIELD_NAME = "ARCH"; - private final static String OS_FIELD_NAME = "OS"; + private static final String ARCH_FIELD_NAME = "ARCH"; + private static final String OS_FIELD_NAME = "OS"; - private final static String MUSL_ENVIRONMENT_FIELD_NAME = "MUSL_ENVIRONMENT"; - private final static String MUSL_LIBC_FIELD_NAME = "MUSL_LIBC"; + private static final String MUSL_ENVIRONMENT_FIELD_NAME = "MUSL_ENVIRONMENT"; + private static final String MUSL_LIBC_FIELD_NAME = "MUSL_LIBC"; private static String INITIAL_OS; private static String INITIAL_ARCH; @@ -255,8 +254,7 @@ public void resolveIsMuslLibc() { assertThat(Environment.initIsMuslLibc()).isFalse(); } - private void setEnvironmentClassFields(String osName, - String osArch) { + private void setEnvironmentClassFields(final String osName, final String osArch) { setEnvironmentClassField(OS_FIELD_NAME, osName); setEnvironmentClassField(ARCH_FIELD_NAME, osArch); } @@ -270,7 +268,7 @@ public static void restoreState() { } @SuppressWarnings("unchecked") - private static T getEnvironmentClassField(String fieldName) { + private static T getEnvironmentClassField(final String fieldName) { final Field field; try { field = Environment.class.getDeclaredField(fieldName); @@ -286,7 +284,7 @@ private static T getEnvironmentClassField(String fieldName) { } } - private static void setEnvironmentClassField(String fieldName, Object value) { + private static void setEnvironmentClassField(final String fieldName, final Object value) { final Field field; try { field = Environment.class.getDeclaredField(fieldName); diff --git a/java/src/test/java/org/rocksdb/util/HeapByteBufferAllocator.java b/java/src/test/java/org/forstdb/util/HeapByteBufferAllocator.java similarity index 95% rename from java/src/test/java/org/rocksdb/util/HeapByteBufferAllocator.java rename to java/src/test/java/org/forstdb/util/HeapByteBufferAllocator.java index ad6b8f6f4..46da1826b 100644 --- a/java/src/test/java/org/rocksdb/util/HeapByteBufferAllocator.java +++ b/java/src/test/java/org/forstdb/util/HeapByteBufferAllocator.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import java.nio.ByteBuffer; diff --git a/java/src/test/java/org/rocksdb/util/IntComparatorTest.java b/java/src/test/java/org/forstdb/util/IntComparatorTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/util/IntComparatorTest.java rename to java/src/test/java/org/forstdb/util/IntComparatorTest.java index dd3288513..f9c4e746d 100644 --- a/java/src/test/java/org/rocksdb/util/IntComparatorTest.java +++ b/java/src/test/java/org/forstdb/util/IntComparatorTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -14,7 +14,7 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; -import org.rocksdb.*; +import org.forstdb.*; import java.nio.ByteBuffer; import java.nio.file.*; diff --git a/java/src/test/java/org/rocksdb/util/JNIComparatorTest.java b/java/src/test/java/org/forstdb/util/JNIComparatorTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/util/JNIComparatorTest.java rename to java/src/test/java/org/forstdb/util/JNIComparatorTest.java index a962b8d78..78e13f5fd 100644 --- a/java/src/test/java/org/rocksdb/util/JNIComparatorTest.java +++ b/java/src/test/java/org/forstdb/util/JNIComparatorTest.java @@ -4,7 +4,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import org.junit.ClassRule; import org.junit.Rule; @@ -14,7 +14,7 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; -import org.rocksdb.*; +import org.forstdb.*; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/java/src/test/java/org/rocksdb/util/ReverseBytewiseComparatorIntTest.java b/java/src/test/java/org/forstdb/util/ReverseBytewiseComparatorIntTest.java similarity index 99% rename from java/src/test/java/org/rocksdb/util/ReverseBytewiseComparatorIntTest.java rename to java/src/test/java/org/forstdb/util/ReverseBytewiseComparatorIntTest.java index ca08d9de1..ebf98d11b 100644 --- a/java/src/test/java/org/rocksdb/util/ReverseBytewiseComparatorIntTest.java +++ b/java/src/test/java/org/forstdb/util/ReverseBytewiseComparatorIntTest.java @@ -3,7 +3,7 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -14,7 +14,7 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; -import org.rocksdb.*; +import org.forstdb.*; import java.nio.ByteBuffer; import java.nio.file.FileSystems; diff --git a/java/src/test/java/org/rocksdb/util/SizeUnitTest.java b/java/src/test/java/org/forstdb/util/SizeUnitTest.java similarity index 97% rename from java/src/test/java/org/rocksdb/util/SizeUnitTest.java rename to java/src/test/java/org/forstdb/util/SizeUnitTest.java index 990aa5f47..190e445d6 100644 --- a/java/src/test/java/org/rocksdb/util/SizeUnitTest.java +++ b/java/src/test/java/org/forstdb/util/SizeUnitTest.java @@ -2,7 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import org.junit.Test; diff --git a/java/src/test/java/org/rocksdb/util/TestUtil.java b/java/src/test/java/org/forstdb/util/TestUtil.java similarity index 93% rename from java/src/test/java/org/rocksdb/util/TestUtil.java rename to java/src/test/java/org/forstdb/util/TestUtil.java index e4f490c8e..a84d97b92 100644 --- a/java/src/test/java/org/rocksdb/util/TestUtil.java +++ b/java/src/test/java/org/forstdb/util/TestUtil.java @@ -3,15 +3,15 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). -package org.rocksdb.util; +package org.forstdb.util; import static java.nio.charset.StandardCharsets.UTF_8; import java.nio.ByteBuffer; import java.util.Random; -import org.rocksdb.CompactionPriority; -import org.rocksdb.Options; -import org.rocksdb.WALRecoveryMode; +import org.forstdb.CompactionPriority; +import org.forstdb.Options; +import org.forstdb.WALRecoveryMode; /** * General test utilities. diff --git a/java/src/test/java/org/rocksdb/util/WriteBatchGetter.java b/java/src/test/java/org/forstdb/util/WriteBatchGetter.java similarity index 97% rename from java/src/test/java/org/rocksdb/util/WriteBatchGetter.java rename to java/src/test/java/org/forstdb/util/WriteBatchGetter.java index 2efa16473..3230eed62 100644 --- a/java/src/test/java/org/rocksdb/util/WriteBatchGetter.java +++ b/java/src/test/java/org/forstdb/util/WriteBatchGetter.java @@ -1,8 +1,8 @@ // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -package org.rocksdb.util; +package org.forstdb.util; -import org.rocksdb.RocksDBException; -import org.rocksdb.WriteBatch; +import org.forstdb.RocksDBException; +import org.forstdb.WriteBatch; import java.util.Arrays; diff --git a/java/src/test/java/org/rocksdb/OptionsUtilTest.java b/java/src/test/java/org/rocksdb/OptionsUtilTest.java deleted file mode 100644 index c2975eadc..000000000 --- a/java/src/test/java/org/rocksdb/OptionsUtilTest.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). - -package org.rocksdb; - -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.util.*; - -import static org.assertj.core.api.Assertions.assertThat; - -public class OptionsUtilTest { - @ClassRule - public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = new RocksNativeLibraryResource(); - - @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); - - enum TestAPI { LOAD_LATEST_OPTIONS, LOAD_OPTIONS_FROM_FILE } - - @Test - public void loadLatestOptions() throws RocksDBException { - verifyOptions(TestAPI.LOAD_LATEST_OPTIONS); - } - - @Test - public void loadOptionsFromFile() throws RocksDBException { - verifyOptions(TestAPI.LOAD_OPTIONS_FROM_FILE); - } - - @Test - public void getLatestOptionsFileName() throws RocksDBException { - final String dbPath = dbFolder.getRoot().getAbsolutePath(); - try (final Options options = new Options().setCreateIfMissing(true); - final RocksDB db = RocksDB.open(options, dbPath)) { - assertThat(db).isNotNull(); - } - - final String fName = OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); - assertThat(fName).isNotNull(); - assert (fName.startsWith("OPTIONS-")); - // System.out.println("latest options fileName: " + fName); - } - - private void verifyOptions(final TestAPI apiType) throws RocksDBException { - final String dbPath = dbFolder.getRoot().getAbsolutePath(); - final Options options = new Options() - .setCreateIfMissing(true) - .setParanoidChecks(false) - .setMaxOpenFiles(478) - .setDelayedWriteRate(1234567L); - final ColumnFamilyOptions baseDefaultCFOpts = new ColumnFamilyOptions(); - final byte[] secondCFName = "new_cf".getBytes(); - final ColumnFamilyOptions baseSecondCFOpts = - new ColumnFamilyOptions() - .setWriteBufferSize(70 * 1024) - .setMaxWriteBufferNumber(7) - .setMaxBytesForLevelBase(53 * 1024 * 1024) - .setLevel0FileNumCompactionTrigger(3) - .setLevel0SlowdownWritesTrigger(51) - .setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); - - // Create a database with a new column family - try (final RocksDB db = RocksDB.open(options, dbPath)) { - assertThat(db).isNotNull(); - - // create column family - try (final ColumnFamilyHandle columnFamilyHandle = - db.createColumnFamily(new ColumnFamilyDescriptor(secondCFName, baseSecondCFOpts))) { - assert(columnFamilyHandle != null); - } - } - - // Read the options back and verify - final DBOptions dbOptions = new DBOptions(); - final ConfigOptions configOptions = - new ConfigOptions().setIgnoreUnknownOptions(false).setInputStringsEscaped(true).setEnv( - Env.getDefault()); - final List cfDescs = new ArrayList<>(); - String path = dbPath; - if (apiType == TestAPI.LOAD_LATEST_OPTIONS) { - OptionsUtil.loadLatestOptions(configOptions, path, dbOptions, cfDescs); - } else if (apiType == TestAPI.LOAD_OPTIONS_FROM_FILE) { - path = dbPath + "/" + OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); - OptionsUtil.loadOptionsFromFile(configOptions, path, dbOptions, cfDescs); - } - - assertThat(dbOptions.createIfMissing()).isEqualTo(options.createIfMissing()); - assertThat(dbOptions.paranoidChecks()).isEqualTo(options.paranoidChecks()); - assertThat(dbOptions.maxOpenFiles()).isEqualTo(options.maxOpenFiles()); - assertThat(dbOptions.delayedWriteRate()).isEqualTo(options.delayedWriteRate()); - - assertThat(cfDescs.size()).isEqualTo(2); - assertThat(cfDescs.get(0)).isNotNull(); - assertThat(cfDescs.get(1)).isNotNull(); - assertThat(cfDescs.get(0).getName()).isEqualTo(RocksDB.DEFAULT_COLUMN_FAMILY); - assertThat(cfDescs.get(1).getName()).isEqualTo(secondCFName); - - final ColumnFamilyOptions defaultCFOpts = cfDescs.get(0).getOptions(); - assertThat(defaultCFOpts.writeBufferSize()).isEqualTo(baseDefaultCFOpts.writeBufferSize()); - assertThat(defaultCFOpts.maxWriteBufferNumber()) - .isEqualTo(baseDefaultCFOpts.maxWriteBufferNumber()); - assertThat(defaultCFOpts.maxBytesForLevelBase()) - .isEqualTo(baseDefaultCFOpts.maxBytesForLevelBase()); - assertThat(defaultCFOpts.level0FileNumCompactionTrigger()) - .isEqualTo(baseDefaultCFOpts.level0FileNumCompactionTrigger()); - assertThat(defaultCFOpts.level0SlowdownWritesTrigger()) - .isEqualTo(baseDefaultCFOpts.level0SlowdownWritesTrigger()); - assertThat(defaultCFOpts.bottommostCompressionType()) - .isEqualTo(baseDefaultCFOpts.bottommostCompressionType()); - - final ColumnFamilyOptions secondCFOpts = cfDescs.get(1).getOptions(); - assertThat(secondCFOpts.writeBufferSize()).isEqualTo(baseSecondCFOpts.writeBufferSize()); - assertThat(secondCFOpts.maxWriteBufferNumber()) - .isEqualTo(baseSecondCFOpts.maxWriteBufferNumber()); - assertThat(secondCFOpts.maxBytesForLevelBase()) - .isEqualTo(baseSecondCFOpts.maxBytesForLevelBase()); - assertThat(secondCFOpts.level0FileNumCompactionTrigger()) - .isEqualTo(baseSecondCFOpts.level0FileNumCompactionTrigger()); - assertThat(secondCFOpts.level0SlowdownWritesTrigger()) - .isEqualTo(baseSecondCFOpts.level0SlowdownWritesTrigger()); - assertThat(secondCFOpts.bottommostCompressionType()) - .isEqualTo(baseSecondCFOpts.bottommostCompressionType()); - } -} diff --git a/logging/auto_roll_logger.cc b/logging/auto_roll_logger.cc index 9e9ad45ae..595e0d246 100644 --- a/logging/auto_roll_logger.cc +++ b/logging/auto_roll_logger.cc @@ -280,7 +280,7 @@ Status CreateLoggerFromOptions(const std::string& dbname, Env* env = options.env; std::string db_absolute_path; Status s = env->GetAbsolutePath(dbname, &db_absolute_path); - TEST_SYNC_POINT_CALLBACK("rocksdb::CreateLoggerFromOptions:AfterGetPath", &s); + TEST_SYNC_POINT_CALLBACK("forstdb::CreateLoggerFromOptions:AfterGetPath", &s); if (!s.ok()) { return s; } diff --git a/logging/auto_roll_logger_test.cc b/logging/auto_roll_logger_test.cc index 3d0ec1763..344fea96e 100644 --- a/logging/auto_roll_logger_test.cc +++ b/logging/auto_roll_logger_test.cc @@ -574,8 +574,8 @@ TEST_F(AutoRollLoggerTest, Close) { static std::vector GetOldFileNames(const std::string& path) { std::vector ret; - const std::string dirname = path.substr(/*start=*/0, path.find_last_of("/")); - const std::string fname = path.substr(path.find_last_of("/") + 1); + const std::string dirname = path.substr(/*start=*/0, path.find_last_of('/')); + const std::string fname = path.substr(path.find_last_of('/') + 1); std::vector children; EXPECT_OK(Env::Default()->GetChildren(dirname, &children)); diff --git a/logging/env_logger_test.cc b/logging/env_logger_test.cc index 467ab064f..21db8b658 100644 --- a/logging/env_logger_test.cc +++ b/logging/env_logger_test.cc @@ -138,7 +138,7 @@ TEST_F(EnvLoggerTest, ConcurrentLogging) { const int kNumThreads = 5; // Create threads. for (int ii = 0; ii < kNumThreads; ++ii) { - threads.push_back(port::Thread(cb)); + threads.emplace_back(cb); } // Wait for them to complete. diff --git a/memory/arena_test.cc b/memory/arena_test.cc index 21bf7ed62..8db761f68 100644 --- a/memory/arena_test.cc +++ b/memory/arena_test.cc @@ -170,7 +170,7 @@ static void SimpleTest(size_t huge_page_size) { r[b] = i % 256; } bytes += s; - allocated.push_back(std::make_pair(s, r)); + allocated.emplace_back(s, r); ASSERT_GE(arena.ApproximateMemoryUsage(), bytes); if (i > N / 10) { ASSERT_LE(arena.ApproximateMemoryUsage(), bytes * 1.10); @@ -219,21 +219,28 @@ size_t PopMinorPageFaultCount() { TEST(MmapTest, AllocateLazyZeroed) { // Doesn't have to be page aligned - constexpr size_t len = 1234567; - MemMapping m = MemMapping::AllocateLazyZeroed(len); - auto arr = static_cast(m.Get()); + constexpr size_t len = 1234567; // in bytes + constexpr size_t count = len / 8; // in uint64_t objects + // Implicit conversion move + TypedMemMapping pre_arr = MemMapping::AllocateLazyZeroed(len); + // Move from same type + TypedMemMapping arr = std::move(pre_arr); - // Should generally work - ASSERT_NE(arr, nullptr); + ASSERT_NE(arr.Get(), nullptr); + ASSERT_EQ(arr.Get(), &arr[0]); + ASSERT_EQ(arr.Get(), arr.MemMapping::Get()); + + ASSERT_EQ(arr.Length(), len); + ASSERT_EQ(arr.Count(), count); // Start counting page faults PopMinorPageFaultCount(); // Access half of the allocation size_t i = 0; - for (; i < len / 2; ++i) { + for (; i < count / 2; ++i) { ASSERT_EQ(arr[i], 0); - arr[i] = static_cast(i & 255); + arr[i] = i; } // Appropriate page faults (maybe more) @@ -241,9 +248,9 @@ TEST(MmapTest, AllocateLazyZeroed) { ASSERT_GE(faults, len / 2 / port::kPageSize); // Access rest of the allocation - for (; i < len; ++i) { + for (; i < count; ++i) { ASSERT_EQ(arr[i], 0); - arr[i] = static_cast(i & 255); + arr[i] = i; } // Appropriate page faults (maybe more) @@ -251,8 +258,8 @@ TEST(MmapTest, AllocateLazyZeroed) { ASSERT_GE(faults, len / 2 / port::kPageSize); // Verify data - for (i = 0; i < len; ++i) { - ASSERT_EQ(arr[i], static_cast(i & 255)); + for (i = 0; i < count; ++i) { + ASSERT_EQ(arr[i], i); } } diff --git a/memory/jemalloc_nodump_allocator.cc b/memory/jemalloc_nodump_allocator.cc index d05248224..9bcd679ae 100644 --- a/memory/jemalloc_nodump_allocator.cc +++ b/memory/jemalloc_nodump_allocator.cc @@ -63,7 +63,7 @@ bool JemallocNodumpAllocator::IsSupported(std::string* why) { } JemallocNodumpAllocator::JemallocNodumpAllocator( - JemallocAllocatorOptions& options) + const JemallocAllocatorOptions& options) : options_(options) #ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR , @@ -124,7 +124,8 @@ uint32_t JemallocNodumpAllocator::GetArenaIndex() const { // to make Random thread-safe and prevent cacheline bouncing. Whether this is // worthwhile is still an open question. thread_local Random tl_random(next_seed.fetch_add(1)); - return arena_indexes_[FastRange32(tl_random.Next(), arena_indexes_.size())]; + return arena_indexes_[FastRange32( + tl_random.Next(), static_cast(arena_indexes_.size()))]; } Status JemallocNodumpAllocator::InitializeArenas() { @@ -282,7 +283,7 @@ void JemallocNodumpAllocator::DestroyThreadSpecificCache(void* ptr) { #endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR Status NewJemallocNodumpAllocator( - JemallocAllocatorOptions& options, + const JemallocAllocatorOptions& options, std::shared_ptr* memory_allocator) { if (memory_allocator == nullptr) { return Status::InvalidArgument("memory_allocator must be non-null."); diff --git a/memory/jemalloc_nodump_allocator.h b/memory/jemalloc_nodump_allocator.h index 2bdbaeb32..c7ab0d6d1 100644 --- a/memory/jemalloc_nodump_allocator.h +++ b/memory/jemalloc_nodump_allocator.h @@ -30,7 +30,7 @@ namespace ROCKSDB_NAMESPACE { // arena mutexes. class JemallocNodumpAllocator : public BaseMemoryAllocator { public: - explicit JemallocNodumpAllocator(JemallocAllocatorOptions& options); + explicit JemallocNodumpAllocator(const JemallocAllocatorOptions& options); #ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR ~JemallocNodumpAllocator(); #endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR diff --git a/memtable/alloc_tracker.cc b/memtable/alloc_tracker.cc index 4c6d35431..d780df0bf 100644 --- a/memtable/alloc_tracker.cc +++ b/memtable/alloc_tracker.cc @@ -7,7 +7,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include +#include #include "memory/allocator.h" #include "memory/arena.h" diff --git a/memtable/hash_linklist_rep.cc b/memtable/hash_linklist_rep.cc index 9e60f9be3..a75587917 100644 --- a/memtable/hash_linklist_rep.cc +++ b/memtable/hash_linklist_rep.cc @@ -81,7 +81,7 @@ struct Node { void NoBarrier_SetNext(Node* x) { next_.store(x, std::memory_order_relaxed); } // Needed for placement new below which is fine - Node() {} + Node() = default; private: std::atomic next_; @@ -265,7 +265,7 @@ class HashLinkListRep : public MemTableRep { explicit FullListIterator(MemtableSkipList* list, Allocator* allocator) : iter_(list), full_list_(list), allocator_(allocator) {} - ~FullListIterator() override {} + ~FullListIterator() override = default; // Returns true iff the iterator is positioned at a valid node. bool Valid() const override { return iter_.Valid(); } @@ -332,7 +332,7 @@ class HashLinkListRep : public MemTableRep { head_(head), node_(nullptr) {} - ~LinkListIterator() override {} + ~LinkListIterator() override = default; // Returns true iff the iterator is positioned at a valid node. bool Valid() const override { return node_ != nullptr; } @@ -482,7 +482,7 @@ class HashLinkListRep : public MemTableRep { // This is used when there wasn't a bucket. It is cheaper than // instantiating an empty bucket over which to iterate. public: - EmptyIterator() {} + EmptyIterator() = default; bool Valid() const override { return false; } const char* key() const override { assert(false); @@ -526,7 +526,7 @@ HashLinkListRep::HashLinkListRep( } } -HashLinkListRep::~HashLinkListRep() {} +HashLinkListRep::~HashLinkListRep() = default; KeyHandle HashLinkListRep::Allocate(const size_t len, char** buf) { char* mem = allocator_->AllocateAligned(sizeof(Node) + len); diff --git a/memtable/hash_skiplist_rep.cc b/memtable/hash_skiplist_rep.cc index 15ff4f071..aed1580c4 100644 --- a/memtable/hash_skiplist_rep.cc +++ b/memtable/hash_skiplist_rep.cc @@ -208,7 +208,7 @@ class HashSkipListRep : public MemTableRep { // This is used when there wasn't a bucket. It is cheaper than // instantiating an empty bucket over which to iterate. public: - EmptyIterator() {} + EmptyIterator() = default; bool Valid() const override { return false; } const char* key() const override { assert(false); @@ -248,7 +248,7 @@ HashSkipListRep::HashSkipListRep(const MemTableRep::KeyComparator& compare, } } -HashSkipListRep::~HashSkipListRep() {} +HashSkipListRep::~HashSkipListRep() = default; HashSkipListRep::Bucket* HashSkipListRep::GetInitializedBucket( const Slice& transformed) { diff --git a/memtable/skiplistrep.cc b/memtable/skiplistrep.cc index c3b4c785d..e615ef9f6 100644 --- a/memtable/skiplistrep.cc +++ b/memtable/skiplistrep.cc @@ -161,7 +161,7 @@ class SkipListRep : public MemTableRep { } } - ~SkipListRep() override {} + ~SkipListRep() override = default; // Iteration over the contents of a skip list class Iterator : public MemTableRep::Iterator { @@ -174,7 +174,7 @@ class SkipListRep : public MemTableRep { const InlineSkipList* list) : iter_(list) {} - ~Iterator() override {} + ~Iterator() override = default; // Returns true iff the iterator is positioned at a valid node. bool Valid() const override { return iter_.Valid(); } @@ -232,7 +232,7 @@ class SkipListRep : public MemTableRep { explicit LookaheadIterator(const SkipListRep& rep) : rep_(rep), iter_(&rep_.skip_list_), prev_(iter_) {} - ~LookaheadIterator() override {} + ~LookaheadIterator() override = default; bool Valid() const override { return iter_.Valid(); } diff --git a/memtable/vectorrep.cc b/memtable/vectorrep.cc index e42ae4439..9b0192cb8 100644 --- a/memtable/vectorrep.cc +++ b/memtable/vectorrep.cc @@ -40,7 +40,7 @@ class VectorRep : public MemTableRep { void Get(const LookupKey& k, void* callback_args, bool (*callback_func)(void* arg, const char* entry)) override; - ~VectorRep() override {} + ~VectorRep() override = default; class Iterator : public MemTableRep::Iterator { class VectorRep* vrep_; @@ -59,7 +59,7 @@ class VectorRep : public MemTableRep { // Initialize an iterator over the specified collection. // The returned iterator is not valid. // explicit Iterator(const MemTableRep* collection); - ~Iterator() override{}; + ~Iterator() override = default; // Returns true iff the iterator is positioned at a valid node. bool Valid() const override; diff --git a/microbench/db_basic_bench.cc b/microbench/db_basic_bench.cc index 20e7182f7..2eca31f10 100644 --- a/microbench/db_basic_bench.cc +++ b/microbench/db_basic_bench.cc @@ -538,6 +538,25 @@ static void ManualFlushArguments(benchmark::internal::Benchmark* b) { BENCHMARK(ManualFlush)->Iterations(1)->Apply(ManualFlushArguments); +// Copied from test_util.cc to not depend on rocksdb_test_lib +// when building microbench binaries. +static Slice CompressibleString(Random* rnd, double compressed_fraction, + int len, std::string* dst) { + int raw = static_cast(len * compressed_fraction); + if (raw < 1) { + raw = 1; + } + std::string raw_data = rnd->RandomBinaryString(raw); + + // Duplicate the random data until we have filled "len" bytes + dst->clear(); + while (dst->size() < (unsigned int)len) { + dst->append(raw_data); + } + dst->resize(len); + return Slice(*dst); +} + static void DBGet(benchmark::State& state) { auto compaction_style = static_cast(state.range(0)); uint64_t max_data = state.range(1); @@ -546,6 +565,9 @@ static void DBGet(benchmark::State& state) { bool negative_query = state.range(4); bool enable_filter = state.range(5); bool mmap = state.range(6); + auto compression_type = static_cast(state.range(7)); + bool compression_checksum = static_cast(state.range(8)); + bool no_blockcache = state.range(9); uint64_t key_num = max_data / per_key_size; // setup DB @@ -568,6 +590,13 @@ static void DBGet(benchmark::State& state) { table_options.no_block_cache = true; table_options.block_restart_interval = 1; } + options.compression = compression_type; + options.compression_opts.checksum = compression_checksum; + if (no_blockcache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = NewLRUCache(100 << 20); + } options.table_factory.reset(NewBlockBasedTableFactory(table_options)); auto rnd = Random(301 + state.thread_index()); @@ -581,9 +610,10 @@ static void DBGet(benchmark::State& state) { // number. auto wo = WriteOptions(); wo.disableWAL = true; + std::string val; for (uint64_t i = 0; i < key_num; i++) { - Status s = db->Put(wo, kg_seq.Next(), - rnd.RandomString(static_cast(per_key_size))); + CompressibleString(&rnd, 0.5, static_cast(per_key_size), &val); + Status s = db->Put(wo, kg_seq.Next(), val); if (!s.ok()) { state.SkipWithError(s.ToString().c_str()); } @@ -641,14 +671,23 @@ static void DBGet(benchmark::State& state) { static void DBGetArguments(benchmark::internal::Benchmark* b) { for (int comp_style : {kCompactionStyleLevel, kCompactionStyleUniversal, kCompactionStyleFIFO}) { - for (int64_t max_data : {128l << 20, 512l << 20}) { + for (int64_t max_data : {1l << 20, 128l << 20, 512l << 20}) { for (int64_t per_key_size : {256, 1024}) { for (bool enable_statistics : {false, true}) { for (bool negative_query : {false, true}) { for (bool enable_filter : {false, true}) { for (bool mmap : {false, true}) { - b->Args({comp_style, max_data, per_key_size, enable_statistics, - negative_query, enable_filter, mmap}); + for (int compression_type : + {kNoCompression /* 0x0 */, kZSTD /* 0x7 */}) { + for (bool compression_checksum : {false, true}) { + for (bool no_blockcache : {false, true}) { + b->Args({comp_style, max_data, per_key_size, + enable_statistics, negative_query, enable_filter, + mmap, compression_type, compression_checksum, + no_blockcache}); + } + } + } } } } @@ -657,11 +696,13 @@ static void DBGetArguments(benchmark::internal::Benchmark* b) { } } b->ArgNames({"comp_style", "max_data", "per_key_size", "enable_statistics", - "negative_query", "enable_filter", "mmap"}); + "negative_query", "enable_filter", "mmap", "compression_type", + "compression_checksum", "no_blockcache"}); } -BENCHMARK(DBGet)->Threads(1)->Apply(DBGetArguments); -BENCHMARK(DBGet)->Threads(8)->Apply(DBGetArguments); +static const uint64_t DBGetNum = 10000l; +BENCHMARK(DBGet)->Threads(1)->Iterations(DBGetNum)->Apply(DBGetArguments); +BENCHMARK(DBGet)->Threads(8)->Iterations(DBGetNum / 8)->Apply(DBGetArguments); static void SimpleGetWithPerfContext(benchmark::State& state) { // setup DB @@ -1555,8 +1596,7 @@ static void RandomAccessFileReaderRead(benchmark::State& state) { uint64_t idx = 0; for (auto _ : state) { s = readers[idx++ % kFileNum]->Read(io_options, 0, kDefaultPageSize / 3, - &result, scratch.get(), nullptr, - Env::IO_TOTAL); + &result, scratch.get(), nullptr); if (!s.ok()) { state.SkipWithError(s.ToString().c_str()); } diff --git a/monitoring/histogram.cc b/monitoring/histogram.cc index 61bc6c140..bc70f3902 100644 --- a/monitoring/histogram.cc +++ b/monitoring/histogram.cc @@ -9,12 +9,11 @@ #include "monitoring/histogram.h" -#include - #include #include #include #include +#include #include "port/port.h" #include "util/cast_util.h" @@ -45,10 +44,11 @@ HistogramBucketMapper::HistogramBucketMapper() { size_t HistogramBucketMapper::IndexForValue(const uint64_t value) const { auto beg = bucketValues_.begin(); auto end = bucketValues_.end(); - if (value >= maxBucketValue_) + if (value >= maxBucketValue_) { return end - beg - 1; // bucketValues_.size() - 1 - else + } else { return std::lower_bound(beg, end, value) - beg; + } } namespace { @@ -147,8 +147,12 @@ double HistogramStat::Percentile(double p) const { double r = left_point + (right_point - left_point) * pos; uint64_t cur_min = min(); uint64_t cur_max = max(); - if (r < cur_min) r = static_cast(cur_min); - if (r > cur_max) r = static_cast(cur_max); + if (r < cur_min) { + r = static_cast(cur_min); + } + if (r > cur_max) { + r = static_cast(cur_max); + } return r; } } @@ -158,7 +162,9 @@ double HistogramStat::Percentile(double p) const { double HistogramStat::Average() const { uint64_t cur_num = num(); uint64_t cur_sum = sum(); - if (cur_num == 0) return 0; + if (cur_num == 0) { + return 0; + } return static_cast(cur_sum) / static_cast(cur_num); } @@ -193,12 +199,16 @@ std::string HistogramStat::ToString() const { Percentile(99.99)); r.append(buf); r.append("------------------------------------------------------\n"); - if (cur_num == 0) return r; // all buckets are empty + if (cur_num == 0) { + return r; // all buckets are empty + } const double mult = 100.0 / cur_num; uint64_t cumulative_sum = 0; for (unsigned int b = 0; b < num_buckets_; b++) { uint64_t bucket_value = bucket_at(b); - if (bucket_value <= 0.0) continue; + if (bucket_value <= 0.0) { + continue; + } cumulative_sum += bucket_value; snprintf(buf, sizeof(buf), "%c %7" PRIu64 ", %7" PRIu64 " ] %8" PRIu64 " %7.3f%% %7.3f%% ", diff --git a/monitoring/histogram_windowing.cc b/monitoring/histogram_windowing.cc index c41ae8a03..726231a71 100644 --- a/monitoring/histogram_windowing.cc +++ b/monitoring/histogram_windowing.cc @@ -34,7 +34,7 @@ HistogramWindowingImpl::HistogramWindowingImpl(uint64_t num_windows, Clear(); } -HistogramWindowingImpl::~HistogramWindowingImpl() {} +HistogramWindowingImpl::~HistogramWindowingImpl() = default; void HistogramWindowingImpl::Clear() { std::lock_guard lock(mutex_); @@ -159,7 +159,9 @@ void HistogramWindowingImpl::SwapHistoryBucket() { for (unsigned int i = 0; i < num_windows_; i++) { if (i != next_window) { uint64_t m = window_stats_[i].min(); - if (m < new_min) new_min = m; + if (m < new_min) { + new_min = m; + } } } stats_.min_.store(new_min, std::memory_order_relaxed); @@ -170,7 +172,9 @@ void HistogramWindowingImpl::SwapHistoryBucket() { for (unsigned int i = 0; i < num_windows_; i++) { if (i != next_window) { uint64_t m = window_stats_[i].max(); - if (m > new_max) new_max = m; + if (m > new_max) { + new_max = m; + } } } stats_.max_.store(new_max, std::memory_order_relaxed); diff --git a/monitoring/in_memory_stats_history.cc b/monitoring/in_memory_stats_history.cc index 568d8ec13..7b7c8db83 100644 --- a/monitoring/in_memory_stats_history.cc +++ b/monitoring/in_memory_stats_history.cc @@ -12,7 +12,7 @@ namespace ROCKSDB_NAMESPACE { -InMemoryStatsHistoryIterator::~InMemoryStatsHistoryIterator() {} +InMemoryStatsHistoryIterator::~InMemoryStatsHistoryIterator() = default; bool InMemoryStatsHistoryIterator::Valid() const { return valid_; } diff --git a/monitoring/instrumented_mutex.h b/monitoring/instrumented_mutex.h index b97d2502e..33e242759 100644 --- a/monitoring/instrumented_mutex.h +++ b/monitoring/instrumented_mutex.h @@ -46,7 +46,7 @@ class InstrumentedMutex { void Unlock() { mutex_.Unlock(); } - void AssertHeld() { mutex_.AssertHeld(); } + void AssertHeld() const { mutex_.AssertHeld(); } private: void LockInternal(); diff --git a/monitoring/perf_context_imp.h b/monitoring/perf_context_imp.h index 5b66ff2ff..f7c9908cc 100644 --- a/monitoring/perf_context_imp.h +++ b/monitoring/perf_context_imp.h @@ -74,7 +74,8 @@ extern thread_local PerfContext perf_context; #define PERF_COUNTER_ADD(metric, value) \ if (perf_level >= PerfLevel::kEnableCount) { \ perf_context.metric += value; \ - } + } \ + static_assert(true, "semicolon required") // Increase metric value #define PERF_COUNTER_BY_LEVEL_ADD(metric, value, level) \ diff --git a/monitoring/perf_level.cc b/monitoring/perf_level.cc index e3507624b..9ba4e0163 100644 --- a/monitoring/perf_level.cc +++ b/monitoring/perf_level.cc @@ -4,7 +4,7 @@ // (found in the LICENSE.Apache file in the root directory). // -#include +#include #include "monitoring/perf_level_imp.h" diff --git a/monitoring/persistent_stats_history.cc b/monitoring/persistent_stats_history.cc index f4c022148..964fe536f 100644 --- a/monitoring/persistent_stats_history.cc +++ b/monitoring/persistent_stats_history.cc @@ -74,7 +74,7 @@ void OptimizeForPersistentStats(ColumnFamilyOptions* cfo) { cfo->compression = kNoCompression; } -PersistentStatsHistoryIterator::~PersistentStatsHistoryIterator() {} +PersistentStatsHistoryIterator::~PersistentStatsHistoryIterator() = default; bool PersistentStatsHistoryIterator::Valid() const { return valid_; } @@ -96,7 +96,7 @@ std::pair parseKey(const Slice& key, uint64_t start_time) { std::pair result; std::string key_str = key.ToString(); - std::string::size_type pos = key_str.find("#"); + std::string::size_type pos = key_str.find('#'); // TODO(Zhongyi): add counters to track parse failures? if (pos == std::string::npos) { result.first = std::numeric_limits::max(); diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index e4a18f943..cc679ec0a 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -111,6 +111,7 @@ const std::vector> TickersNameMap = { {NUMBER_BLOCK_NOT_COMPRESSED, "rocksdb.number.block.not_compressed"}, {MERGE_OPERATION_TOTAL_TIME, "rocksdb.merge.operation.time.nanos"}, {FILTER_OPERATION_TOTAL_TIME, "rocksdb.filter.operation.time.nanos"}, + {COMPACTION_CPU_TOTAL_TIME, "rocksdb.compaction.total.time.cpu_micros"}, {ROW_CACHE_HIT, "rocksdb.row.cache.hit"}, {ROW_CACHE_MISS, "rocksdb.row.cache.miss"}, {READ_AMP_ESTIMATE_USEFUL_BYTES, "rocksdb.read.amp.estimate.useful.bytes"}, @@ -257,6 +258,20 @@ const std::vector> TickersNameMap = { "rocksdb.number.block_compression_rejected"}, {BYTES_DECOMPRESSED_FROM, "rocksdb.bytes.decompressed.from"}, {BYTES_DECOMPRESSED_TO, "rocksdb.bytes.decompressed.to"}, + {READAHEAD_TRIMMED, "rocksdb.readahead.trimmed"}, + {FIFO_MAX_SIZE_COMPACTIONS, "rocksdb.fifo.max.size.compactions"}, + {FIFO_TTL_COMPACTIONS, "rocksdb.fifo.ttl.compactions"}, + {PREFETCH_BYTES, "rocksdb.prefetch.bytes"}, + {PREFETCH_BYTES_USEFUL, "rocksdb.prefetch.bytes.useful"}, + {PREFETCH_HITS, "rocksdb.prefetch.hits"}, + {COMPRESSED_SECONDARY_CACHE_DUMMY_HITS, + "rocksdb.compressed.secondary.cache.dummy.hits"}, + {COMPRESSED_SECONDARY_CACHE_HITS, + "rocksdb.compressed.secondary.cache.hits"}, + {COMPRESSED_SECONDARY_CACHE_PROMOTIONS, + "rocksdb.compressed.secondary.cache.promotions"}, + {COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS, + "rocksdb.compressed.secondary.cache.promotion.skips"}, }; const std::vector> HistogramsNameMap = { @@ -281,6 +296,13 @@ const std::vector> HistogramsNameMap = { {FILE_READ_FLUSH_MICROS, "rocksdb.file.read.flush.micros"}, {FILE_READ_COMPACTION_MICROS, "rocksdb.file.read.compaction.micros"}, {FILE_READ_DB_OPEN_MICROS, "rocksdb.file.read.db.open.micros"}, + {FILE_READ_GET_MICROS, "rocksdb.file.read.get.micros"}, + {FILE_READ_MULTIGET_MICROS, "rocksdb.file.read.multiget.micros"}, + {FILE_READ_DB_ITERATOR_MICROS, "rocksdb.file.read.db.iterator.micros"}, + {FILE_READ_VERIFY_DB_CHECKSUM_MICROS, + "rocksdb.file.read.verify.db.checksum.micros"}, + {FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS, + "rocksdb.file.read.verify.file.checksums.micros"}, {NUM_SUBCOMPACTIONS_SCHEDULED, "rocksdb.num.subcompactions.scheduled"}, {BYTES_PER_READ, "rocksdb.bytes.per.read"}, {BYTES_PER_WRITE, "rocksdb.bytes.per.write"}, @@ -365,7 +387,7 @@ StatisticsImpl::StatisticsImpl(std::shared_ptr stats) RegisterOptions("StatisticsOptions", &stats_, &stats_type_info); } -StatisticsImpl::~StatisticsImpl() {} +StatisticsImpl::~StatisticsImpl() = default; uint64_t StatisticsImpl::getTickerCount(uint32_t tickerType) const { MutexLock lock(&aggregate_lock_); @@ -524,7 +546,9 @@ std::string StatisticsImpl::ToString() const { bool StatisticsImpl::getTickerMap( std::map* stats_map) const { assert(stats_map); - if (!stats_map) return false; + if (!stats_map) { + return false; + } stats_map->clear(); MutexLock lock(&aggregate_lock_); for (const auto& t : TickersNameMap) { diff --git a/monitoring/stats_history_test.cc b/monitoring/stats_history_test.cc index cfed7bad7..37db0cfe1 100644 --- a/monitoring/stats_history_test.cc +++ b/monitoring/stats_history_test.cc @@ -185,6 +185,8 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) { TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { constexpr int kPeriodSec = 1; + constexpr int kEstimatedOneSliceSize = 16000; + Options options; options.create_if_missing = true; options.statistics = CreateDBStatistics(); @@ -206,6 +208,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { ASSERT_TRUE(iterator->key() == iterator->value()); } + ASSERT_OK(iterator->status()); delete iterator; ASSERT_OK(Flush()); ASSERT_OK(Delete("sol")); @@ -219,6 +222,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { ASSERT_TRUE(iterator->key() == iterator->value()); } + ASSERT_OK(iterator->status()); delete iterator; ASSERT_OK(Flush()); ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); @@ -242,10 +246,12 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { } size_t stats_history_size = dbfull()->TEST_EstimateInMemoryStatsHistorySize(); ASSERT_GE(slice_count, kIterations - 1); - ASSERT_GE(stats_history_size, 15000); - // capping memory cost at 15000 bytes since one slice is around 10000~15000 - ASSERT_OK(dbfull()->SetDBOptions({{"stats_history_buffer_size", "15000"}})); - ASSERT_EQ(15000, dbfull()->GetDBOptions().stats_history_buffer_size); + ASSERT_GE(stats_history_size, kEstimatedOneSliceSize); + // capping memory cost to roughly one slice's size + ASSERT_OK(dbfull()->SetDBOptions( + {{"stats_history_buffer_size", std::to_string(kEstimatedOneSliceSize)}})); + ASSERT_EQ(kEstimatedOneSliceSize, + dbfull()->GetDBOptions().stats_history_buffer_size); // Wait for stats persist to finish for (int i = 0; i < kIterations; ++i) { @@ -265,9 +271,13 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { } size_t stats_history_size_reopen = dbfull()->TEST_EstimateInMemoryStatsHistorySize(); - // only one slice can fit under the new stats_history_buffer_size - ASSERT_LT(slice_count, 2); - ASSERT_TRUE(stats_history_size_reopen < 15000 && + + // Only one slice can fit under the new stats_history_buffer_size + // + // If `slice_count == 0` when new statistics are added, consider increasing + // `kEstimatedOneSliceSize` + ASSERT_EQ(slice_count, 1); + ASSERT_TRUE(stats_history_size_reopen < 16000 && stats_history_size_reopen > 0); ASSERT_TRUE(stats_count_reopen < stats_count && stats_count_reopen > 0); Close(); @@ -280,6 +290,7 @@ int countkeys(Iterator* iter) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { count++; } + EXPECT_OK(iter->status()); return count; } diff --git a/monitoring/thread_status_impl.cc b/monitoring/thread_status_impl.cc index 9619dfd81..153753682 100644 --- a/monitoring/thread_status_impl.cc +++ b/monitoring/thread_status_impl.cc @@ -67,7 +67,7 @@ const std::string ThreadStatus::MicrosToString(uint64_t micros) { const std::string& ThreadStatus::GetOperationPropertyName( ThreadStatus::OperationType op_type, int i) { - static const std::string empty_str = ""; + static const std::string empty_str; switch (op_type) { case ThreadStatus::OP_COMPACTION: if (i >= NUM_COMPACTION_PROPERTIES) { diff --git a/monitoring/thread_status_util_debug.cc b/monitoring/thread_status_util_debug.cc index 6e4fe8a9f..24d269cbb 100644 --- a/monitoring/thread_status_util_debug.cc +++ b/monitoring/thread_status_util_debug.cc @@ -36,6 +36,20 @@ Env::IOActivity ThreadStatusUtil::TEST_GetExpectedIOActivity( return Env::IOActivity::kCompaction; case ThreadStatus::OperationType::OP_DBOPEN: return Env::IOActivity::kDBOpen; + case ThreadStatus::OperationType::OP_GET: + return Env::IOActivity::kGet; + case ThreadStatus::OperationType::OP_MULTIGET: + return Env::IOActivity::kMultiGet; + case ThreadStatus::OperationType::OP_DBITERATOR: + return Env::IOActivity::kDBIterator; + case ThreadStatus::OperationType::OP_VERIFY_DB_CHECKSUM: + return Env::IOActivity::kVerifyDBChecksum; + case ThreadStatus::OperationType::OP_VERIFY_FILE_CHECKSUMS: + return Env::IOActivity::kVerifyFileChecksums; + case ThreadStatus::OperationType::OP_GETENTITY: + return Env::IOActivity::kGetEntity; + case ThreadStatus::OperationType::OP_MULTIGETENTITY: + return Env::IOActivity::kMultiGetEntity; default: return Env::IOActivity::kUnknown; } diff --git a/options/cf_options.cc b/options/cf_options.cc index ad1e669df..2ca826c98 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -173,6 +173,9 @@ static std::unordered_map {offsetof(struct CompressionOptions, use_zstd_dict_trainer), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"checksum", + {offsetof(struct CompressionOptions, checksum), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, }; static std::unordered_map @@ -414,7 +417,7 @@ static std::unordered_map // value, say, like "23", which would be assigned to // max_table_files_size. if (name == "compaction_options_fifo" && - value.find("=") == std::string::npos) { + value.find('=') == std::string::npos) { // Old format. Parse just a single uint64_t value. auto options = static_cast(addr); options->max_table_files_size = ParseUint64(value); @@ -507,6 +510,10 @@ static std::unordered_map {offsetof(struct MutableCFOptions, memtable_protection_bytes_per_key), OptionType::kUInt32T, OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"bottommost_file_compaction_delay", + {offsetof(struct MutableCFOptions, bottommost_file_compaction_delay), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, {"block_protection_bytes_per_key", {offsetof(struct MutableCFOptions, block_protection_bytes_per_key), OptionType::kUInt8T, OptionVerificationType::kNormal, @@ -522,7 +529,7 @@ static std::unordered_map // This is to handle backward compatibility, where // compression_options was a ":" separated list. if (name == kOptNameCompOpts && - value.find("=") == std::string::npos) { + value.find('=') == std::string::npos) { auto* compression = static_cast(addr); return ParseCompressionOptions(value, name, *compression); } else { @@ -542,7 +549,7 @@ static std::unordered_map // This is to handle backward compatibility, where // compression_options was a ":" separated list. if (name == kOptNameBMCompOpts && - value.find("=") == std::string::npos) { + value.find('=') == std::string::npos) { auto* compression = static_cast(addr); return ParseCompressionOptions(value, name, *compression); } else { @@ -552,6 +559,11 @@ static std::unordered_map } })}, // End special case properties + {"memtable_max_range_deletions", + {offsetof(struct MutableCFOptions, memtable_max_range_deletions), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + }; static std::unordered_map @@ -595,6 +607,10 @@ static std::unordered_map {offsetof(struct ImmutableCFOptions, force_consistency_checks), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"default_temperature", + {offsetof(struct ImmutableCFOptions, default_temperature), + OptionType::kTemperature, OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever}}, {"preclude_last_level_data_seconds", {offsetof(struct ImmutableCFOptions, preclude_last_level_data_seconds), OptionType::kUInt64T, OptionVerificationType::kNormal, @@ -611,7 +627,7 @@ static std::unordered_map {offsetof(struct ImmutableCFOptions, max_write_buffer_number_to_maintain), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone, nullptr}}, {"max_write_buffer_size_to_maintain", {offsetof(struct ImmutableCFOptions, max_write_buffer_size_to_maintain), @@ -620,7 +636,7 @@ static std::unordered_map {"min_write_buffer_number_to_merge", {offsetof(struct ImmutableCFOptions, min_write_buffer_number_to_merge), OptionType::kInt, OptionVerificationType::kNormal, - OptionTypeFlags::kNone, 0}}, + OptionTypeFlags::kNone, nullptr}}, {"num_levels", {offsetof(struct ImmutableCFOptions, num_levels), OptionType::kInt, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, @@ -642,19 +658,12 @@ static std::unordered_map // it's a const pointer of const Comparator* const auto* ptr = static_cast(addr); - // Since the user-specified comparator will be wrapped by - // InternalKeyComparator, we should persist the - // user-specified one instead of InternalKeyComparator. if (*ptr == nullptr) { *value = kNullptrString; } else if (opts.mutable_options_only) { *value = ""; } else { - const Comparator* root_comp = (*ptr)->GetRootComparator(); - if (root_comp == nullptr) { - root_comp = (*ptr); - } - *value = root_comp->ToString(opts); + *value = (*ptr)->ToString(opts); } return Status::OK(); })}, @@ -941,6 +950,7 @@ ImmutableCFOptions::ImmutableCFOptions(const ColumnFamilyOptions& cf_options) num_levels(cf_options.num_levels), optimize_filters_for_hits(cf_options.optimize_filters_for_hits), force_consistency_checks(cf_options.force_consistency_checks), + default_temperature(cf_options.default_temperature), preclude_last_level_data_seconds( cf_options.preclude_last_level_data_seconds), preserve_internal_time_seconds(cf_options.preserve_internal_time_seconds), @@ -1112,6 +1122,8 @@ void MutableCFOptions::Dump(Logger* log) const { ROCKS_LOG_INFO(log, " experimental_mempurge_threshold: %f", experimental_mempurge_threshold); + ROCKS_LOG_INFO(log, " bottommost_file_compaction_delay: %" PRIu32, + bottommost_file_compaction_delay); // Universal Compaction Options ROCKS_LOG_INFO(log, "compaction_options_universal.size_ratio : %d", diff --git a/options/cf_options.h b/options/cf_options.h index 37ef54c0c..f42d6b562 100644 --- a/options/cf_options.h +++ b/options/cf_options.h @@ -72,6 +72,8 @@ struct ImmutableCFOptions { bool force_consistency_checks; + Temperature default_temperature; + uint64_t preclude_last_level_data_seconds; uint64_t preserve_internal_time_seconds; @@ -175,7 +177,10 @@ struct MutableCFOptions { block_protection_bytes_per_key(options.block_protection_bytes_per_key), sample_for_compression( options.sample_for_compression), // TODO: is 0 fine here? - compression_per_level(options.compression_per_level) { + compression_per_level(options.compression_per_level), + memtable_max_range_deletions(options.memtable_max_range_deletions), + bottommost_file_compaction_delay( + options.bottommost_file_compaction_delay) { RefreshDerivedOptions(options.num_levels, options.compaction_style); } @@ -224,7 +229,8 @@ struct MutableCFOptions { last_level_temperature(Temperature::kUnknown), memtable_protection_bytes_per_key(0), block_protection_bytes_per_key(0), - sample_for_compression(0) {} + sample_for_compression(0), + memtable_max_range_deletions(0) {} explicit MutableCFOptions(const Options& options); @@ -318,6 +324,8 @@ struct MutableCFOptions { uint64_t sample_for_compression; std::vector compression_per_level; + uint32_t memtable_max_range_deletions; + uint32_t bottommost_file_compaction_delay; // Derived options // Per-level target file size. diff --git a/options/configurable.cc b/options/configurable.cc index 5491336e0..134de99a2 100644 --- a/options/configurable.cc +++ b/options/configurable.cc @@ -37,9 +37,9 @@ Status Configurable::PrepareOptions(const ConfigOptions& opts) { // We ignore the invoke_prepare_options here intentionally, // as if you are here, you must have called PrepareOptions explicitly. Status status = Status::OK(); - for (auto opt_iter : options_) { + for (const auto& opt_iter : options_) { if (opt_iter.type_map != nullptr) { - for (auto map_iter : *(opt_iter.type_map)) { + for (const auto& map_iter : *(opt_iter.type_map)) { auto& opt_info = map_iter.second; if (opt_info.ShouldPrepare()) { status = opt_info.Prepare(opts, map_iter.first, opt_iter.opt_ptr); @@ -56,9 +56,9 @@ Status Configurable::PrepareOptions(const ConfigOptions& opts) { Status Configurable::ValidateOptions(const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts) const { Status status; - for (auto opt_iter : options_) { + for (const auto& opt_iter : options_) { if (opt_iter.type_map != nullptr) { - for (auto map_iter : *(opt_iter.type_map)) { + for (const auto& map_iter : *(opt_iter.type_map)) { auto& opt_info = map_iter.second; if (opt_info.ShouldValidate()) { status = opt_info.Validate(db_opts, cf_opts, map_iter.first, @@ -80,7 +80,7 @@ Status Configurable::ValidateOptions(const DBOptions& db_opts, /*********************************************************************************/ const void* Configurable::GetOptionsPtr(const std::string& name) const { - for (auto o : options_) { + for (const auto& o : options_) { if (o.name == name) { return o.opt_ptr; } @@ -95,7 +95,7 @@ std::string Configurable::GetOptionName(const std::string& opt_name) const { const OptionTypeInfo* ConfigurableHelper::FindOption( const std::vector& options, const std::string& short_name, std::string* opt_name, void** opt_ptr) { - for (auto iter : options) { + for (const auto& iter : options) { if (iter.type_map != nullptr) { const auto opt_info = OptionTypeInfo::Find(short_name, *(iter.type_map), opt_name); @@ -318,21 +318,29 @@ Status ConfigurableHelper::ConfigureSomeOptions( } // End while found one or options remain // Now that we have been through the list, remove any unsupported - for (auto u : unsupported) { + for (const auto& u : unsupported) { auto it = options->find(u); if (it != options->end()) { options->erase(it); } } if (config_options.ignore_unknown_options) { - if (!result.ok()) result.PermitUncheckedError(); - if (!notsup.ok()) notsup.PermitUncheckedError(); + if (!result.ok()) { + result.PermitUncheckedError(); + } + if (!notsup.ok()) { + notsup.PermitUncheckedError(); + } return Status::OK(); } else if (!result.ok()) { - if (!notsup.ok()) notsup.PermitUncheckedError(); + if (!notsup.ok()) { + notsup.PermitUncheckedError(); + } return result; } else if (config_options.ignore_unsupported_options) { - if (!notsup.ok()) notsup.PermitUncheckedError(); + if (!notsup.ok()) { + notsup.PermitUncheckedError(); + } return Status::OK(); } else { return notsup; @@ -374,7 +382,7 @@ Status ConfigurableHelper::ConfigureCustomizableOption( return Status::OK(); } else if (custom == nullptr || !StartsWith(name, custom->GetId() + ".")) { return configurable.ParseOption(copy, opt_info, name, value, opt_ptr); - } else if (value.find("=") != std::string::npos) { + } else if (value.find('=') != std::string::npos) { return custom->ConfigureFromString(copy, value); } else { return custom->ConfigureOption(copy, name, value); diff --git a/options/configurable_test.cc b/options/configurable_test.cc index a03d8f0a5..9284e8622 100644 --- a/options/configurable_test.cc +++ b/options/configurable_test.cc @@ -436,7 +436,7 @@ TEST_F(ConfigurableTest, AliasOptionsTest) { OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, {"alias", {offsetof(struct TestOptions, b), OptionType::kBoolean, - OptionVerificationType::kAlias, OptionTypeFlags::kNone, 0}}}; + OptionVerificationType::kAlias, OptionTypeFlags::kNone, nullptr}}}; std::unique_ptr orig; orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode, &alias_option_info)); @@ -758,7 +758,7 @@ void ConfigurableParamTest::TestConfigureOptions( ASSERT_OK(base->GetOptionNames(config_options, &names)); std::unordered_map unused; bool found_one = false; - for (auto name : names) { + for (const auto& name : names) { std::string value; Status s = base->GetOption(config_options, name, &value); if (s.ok()) { diff --git a/options/configurable_test.h b/options/configurable_test.h index 3d6fe8410..7acac776e 100644 --- a/options/configurable_test.h +++ b/options/configurable_test.h @@ -33,8 +33,8 @@ struct TestOptions { bool b = false; bool d = true; TestEnum e = TestEnum::kTestA; - std::string s = ""; - std::string u = ""; + std::string s; + std::string u; }; static std::unordered_map simple_option_info = { diff --git a/options/customizable_test.cc b/options/customizable_test.cc index d88777793..696f1b25e 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -1230,13 +1230,19 @@ class TestSecondaryCache : public SecondaryCache { static const char* kClassName() { return "Test"; } const char* Name() const override { return kClassName(); } Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/, - const Cache::CacheItemHelper* /*helper*/) override { + const Cache::CacheItemHelper* /*helper*/, + bool /*force_insert*/) override { return Status::NotSupported(); } + Status InsertSaved(const Slice& /*key*/, const Slice& /*saved*/, + CompressionType /*type*/, CacheTier /*source*/) override { + return Status::OK(); + } std::unique_ptr Lookup( const Slice& /*key*/, const Cache::CacheItemHelper* /*helper*/, Cache::CreateContext* /*create_context*/, bool /*wait*/, - bool /*advise_erase*/, bool& kept_in_sec_cache) override { + bool /*advise_erase*/, Statistics* /*stats*/, + bool& kept_in_sec_cache) override { kept_in_sec_cache = true; return nullptr; } @@ -1260,7 +1266,7 @@ class TestStatistics : public StatisticsImpl { class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory { public: - TestFlushBlockPolicyFactory() {} + TestFlushBlockPolicyFactory() = default; static const char* kClassName() { return "TestFlushBlockPolicyFactory"; } const char* Name() const override { return kClassName(); } diff --git a/options/db_options.cc b/options/db_options.cc index d81e72833..2d213f13f 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -129,6 +129,10 @@ static std::unordered_map {offsetof(struct MutableDBOptions, max_background_flushes), OptionType::kInt, OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"daily_offpeak_time_utc", + {offsetof(struct MutableDBOptions, daily_offpeak_time_utc), + OptionType::kString, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, }; static std::unordered_map @@ -222,6 +226,10 @@ static std::unordered_map {offsetof(struct ImmutableDBOptions, flush_verify_memtable_count), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"compaction_verify_record_count", + {offsetof(struct ImmutableDBOptions, compaction_verify_record_count), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"track_and_verify_wals_in_manifest", {offsetof(struct ImmutableDBOptions, track_and_verify_wals_in_manifest), @@ -679,6 +687,7 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) error_if_exists(options.error_if_exists), paranoid_checks(options.paranoid_checks), flush_verify_memtable_count(options.flush_verify_memtable_count), + compaction_verify_record_count(options.compaction_verify_record_count), track_and_verify_wals_in_manifest( options.track_and_verify_wals_in_manifest), verify_sst_unique_id_in_manifest( @@ -771,6 +780,8 @@ void ImmutableDBOptions::Dump(Logger* log) const { paranoid_checks); ROCKS_LOG_HEADER(log, " Options.flush_verify_memtable_count: %d", flush_verify_memtable_count); + ROCKS_LOG_HEADER(log, " Options.compaction_verify_record_count: %d", + compaction_verify_record_count); ROCKS_LOG_HEADER(log, " " "Options.track_and_verify_wals_in_manifest: %d", @@ -787,6 +798,11 @@ void ImmutableDBOptions::Dump(Logger* log) const { max_file_opening_threads); ROCKS_LOG_HEADER(log, " Options.statistics: %p", stats); + if (stats) { + ROCKS_LOG_HEADER( + log, " Options.statistics stats level: %u", + stats->get_stats_level()); + } ROCKS_LOG_HEADER(log, " Options.use_fsync: %d", use_fsync); ROCKS_LOG_HEADER( @@ -999,7 +1015,8 @@ MutableDBOptions::MutableDBOptions(const DBOptions& options) wal_bytes_per_sync(options.wal_bytes_per_sync), strict_bytes_per_sync(options.strict_bytes_per_sync), compaction_readahead_size(options.compaction_readahead_size), - max_background_flushes(options.max_background_flushes) {} + max_background_flushes(options.max_background_flushes), + daily_offpeak_time_utc(options.daily_offpeak_time_utc) {} void MutableDBOptions::Dump(Logger* log) const { ROCKS_LOG_HEADER(log, " Options.max_background_jobs: %d", @@ -1044,6 +1061,8 @@ void MutableDBOptions::Dump(Logger* log) const { compaction_readahead_size); ROCKS_LOG_HEADER(log, " Options.max_background_flushes: %d", max_background_flushes); + ROCKS_LOG_HEADER(log, "Options.daily_offpeak_time_utc: %s", + daily_offpeak_time_utc.c_str()); } Status GetMutableDBOptionsFromStrings( diff --git a/options/db_options.h b/options/db_options.h index 2a9d98b25..701a83feb 100644 --- a/options/db_options.h +++ b/options/db_options.h @@ -25,6 +25,7 @@ struct ImmutableDBOptions { bool error_if_exists; bool paranoid_checks; bool flush_verify_memtable_count; + bool compaction_verify_record_count; bool track_and_verify_wals_in_manifest; bool verify_sst_unique_id_in_manifest; Env* env; @@ -135,6 +136,7 @@ struct MutableDBOptions { bool strict_bytes_per_sync; size_t compaction_readahead_size; int max_background_flushes; + std::string daily_offpeak_time_utc; }; Status GetStringFromMutableDBOptions(const ConfigOptions& config_options, diff --git a/options/offpeak_time_info.cc b/options/offpeak_time_info.cc new file mode 100644 index 000000000..4eaeb6e27 --- /dev/null +++ b/options/offpeak_time_info.cc @@ -0,0 +1,59 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "options/offpeak_time_info.h" + +#include "rocksdb/system_clock.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +OffpeakTimeOption::OffpeakTimeOption() : OffpeakTimeOption("") {} +OffpeakTimeOption::OffpeakTimeOption(const std::string& offpeak_time_string) { + SetFromOffpeakTimeString(offpeak_time_string); +} + +void OffpeakTimeOption::SetFromOffpeakTimeString( + const std::string& offpeak_time_string) { + const int old_start_time = daily_offpeak_start_time_utc; + const int old_end_time = daily_offpeak_end_time_utc; + if (TryParseTimeRangeString(offpeak_time_string, daily_offpeak_start_time_utc, + daily_offpeak_end_time_utc)) { + daily_offpeak_time_utc = offpeak_time_string; + } else { + daily_offpeak_start_time_utc = old_start_time; + daily_offpeak_end_time_utc = old_end_time; + } +} + +OffpeakTimeInfo OffpeakTimeOption::GetOffpeakTimeInfo( + const int64_t& current_time) const { + OffpeakTimeInfo offpeak_time_info; + if (daily_offpeak_start_time_utc == daily_offpeak_end_time_utc) { + return offpeak_time_info; + } + int seconds_since_midnight = static_cast(current_time % kSecondsPerDay); + int seconds_since_midnight_to_nearest_minute = + (seconds_since_midnight / kSecondsPerMinute) * kSecondsPerMinute; + // if the offpeak duration spans overnight (i.e. 23:30 - 4:30 next day) + if (daily_offpeak_start_time_utc > daily_offpeak_end_time_utc) { + offpeak_time_info.is_now_offpeak = + daily_offpeak_start_time_utc <= + seconds_since_midnight_to_nearest_minute || + seconds_since_midnight_to_nearest_minute <= daily_offpeak_end_time_utc; + } else { + offpeak_time_info.is_now_offpeak = + daily_offpeak_start_time_utc <= + seconds_since_midnight_to_nearest_minute && + seconds_since_midnight_to_nearest_minute <= daily_offpeak_end_time_utc; + } + offpeak_time_info.seconds_till_next_offpeak_start = + seconds_since_midnight < daily_offpeak_start_time_utc + ? daily_offpeak_start_time_utc - seconds_since_midnight + : ((daily_offpeak_start_time_utc + kSecondsPerDay) - + seconds_since_midnight); + return offpeak_time_info; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/options/offpeak_time_info.h b/options/offpeak_time_info.h new file mode 100644 index 000000000..75d61abb4 --- /dev/null +++ b/options/offpeak_time_info.h @@ -0,0 +1,36 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +class SystemClock; + +struct OffpeakTimeInfo { + bool is_now_offpeak = false; + int seconds_till_next_offpeak_start = 0; +}; + +struct OffpeakTimeOption { + static constexpr int kSecondsPerDay = 86400; + static constexpr int kSecondsPerHour = 3600; + static constexpr int kSecondsPerMinute = 60; + + OffpeakTimeOption(); + explicit OffpeakTimeOption(const std::string& offpeak_time_string); + std::string daily_offpeak_time_utc = ""; + int daily_offpeak_start_time_utc = 0; + int daily_offpeak_end_time_utc = 0; + + void SetFromOffpeakTimeString(const std::string& offpeak_time_string); + + OffpeakTimeInfo GetOffpeakTimeInfo(const int64_t& current_time) const; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/options/options.cc b/options/options.cc index 1caebdfb2..d96cf4072 100644 --- a/options/options.cc +++ b/options/options.cc @@ -94,6 +94,7 @@ AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options) ttl(options.ttl), periodic_compaction_seconds(options.periodic_compaction_seconds), sample_for_compression(options.sample_for_compression), + default_temperature(options.default_temperature), preclude_last_level_data_seconds( options.preclude_last_level_data_seconds), preserve_internal_time_seconds(options.preserve_internal_time_seconds), @@ -126,7 +127,7 @@ ColumnFamilyOptions::ColumnFamilyOptions() ColumnFamilyOptions::ColumnFamilyOptions(const Options& options) : ColumnFamilyOptions(*static_cast(&options)) {} -DBOptions::DBOptions() {} +DBOptions::DBOptions() = default; DBOptions::DBOptions(const Options& options) : DBOptions(*static_cast(&options)) {} @@ -412,6 +413,17 @@ void ColumnFamilyOptions::Dump(Logger* log) const { ROCKS_LOG_HEADER(log, " Options.periodic_compaction_seconds: %" PRIu64, periodic_compaction_seconds); + const auto& it_temp = temperature_to_string.find(default_temperature); + std::string str_default_temperature; + if (it_temp == temperature_to_string.end()) { + assert(false); + str_default_temperature = "unknown_temperature"; + } else { + str_default_temperature = it_temp->second; + } + ROCKS_LOG_HEADER(log, + " Options.default_temperature: %s", + str_default_temperature.c_str()); ROCKS_LOG_HEADER(log, " Options.preclude_last_level_data_seconds: %" PRIu64, preclude_last_level_data_seconds); ROCKS_LOG_HEADER(log, " Options.preserve_internal_time_seconds: %" PRIu64, @@ -448,8 +460,10 @@ void ColumnFamilyOptions::Dump(Logger* log) const { ? "flush only" : "disabled"); } - ROCKS_LOG_HEADER(log, "Options.experimental_mempurge_threshold: %f", + ROCKS_LOG_HEADER(log, " Options.experimental_mempurge_threshold: %f", experimental_mempurge_threshold); + ROCKS_LOG_HEADER(log, " Options.memtable_max_range_deletions: %d", + memtable_max_range_deletions); } // ColumnFamilyOptions::Dump void Options::Dump(Logger* log) const { diff --git a/options/options_helper.cc b/options/options_helper.cc index abe5053d2..fa5d549c1 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -4,6 +4,7 @@ // (found in the LICENSE.Apache file in the root directory). #include "options/options_helper.h" +#include #include #include #include @@ -45,7 +46,9 @@ Status ValidateOptions(const DBOptions& db_opts, auto db_cfg = DBOptionsAsConfigurable(db_opts); auto cf_cfg = CFOptionsAsConfigurable(cf_opts); s = db_cfg->ValidateOptions(db_opts, cf_opts); - if (s.ok()) s = cf_cfg->ValidateOptions(db_opts, cf_opts); + if (s.ok()) { + s = cf_cfg->ValidateOptions(db_opts, cf_opts); + } return s; } @@ -60,6 +63,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.paranoid_checks = immutable_db_options.paranoid_checks; options.flush_verify_memtable_count = immutable_db_options.flush_verify_memtable_count; + options.compaction_verify_record_count = + immutable_db_options.compaction_verify_record_count; options.track_and_verify_wals_in_manifest = immutable_db_options.track_and_verify_wals_in_manifest; options.verify_sst_unique_id_in_manifest = @@ -176,6 +181,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.lowest_used_cache_tier = immutable_db_options.lowest_used_cache_tier; options.enforce_single_del_contracts = immutable_db_options.enforce_single_del_contracts; + options.daily_offpeak_time_utc = mutable_db_options.daily_offpeak_time_utc; return options; } @@ -208,6 +214,8 @@ void UpdateColumnFamilyOptions(const MutableCFOptions& moptions, moptions.memtable_protection_bytes_per_key; cf_opts->block_protection_bytes_per_key = moptions.block_protection_bytes_per_key; + cf_opts->bottommost_file_compaction_delay = + moptions.bottommost_file_compaction_delay; // Compaction related options cf_opts->disable_auto_compactions = moptions.disable_auto_compactions; @@ -270,6 +278,7 @@ void UpdateColumnFamilyOptions(const MutableCFOptions& moptions, cf_opts->compression_per_level = moptions.compression_per_level; cf_opts->last_level_temperature = moptions.last_level_temperature; cf_opts->bottommost_temperature = moptions.last_level_temperature; + cf_opts->memtable_max_range_deletions = moptions.memtable_max_range_deletions; } void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions, @@ -312,6 +321,7 @@ void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions, ioptions.preserve_internal_time_seconds; cf_opts->persist_user_defined_timestamps = ioptions.persist_user_defined_timestamps; + cf_opts->default_temperature = ioptions.default_temperature; // TODO(yhchiang): find some way to handle the following derived options // * max_file_size @@ -426,6 +436,10 @@ static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type, case OptionType::kSizeT: PutUnaligned(static_cast(opt_address), ParseSizeT(value)); break; + case OptionType::kAtomicInt: + static_cast*>(opt_address) + ->store(ParseInt(value), std::memory_order_release); + break; case OptionType::kString: *static_cast(opt_address) = value; break; @@ -515,6 +529,10 @@ bool SerializeSingleOptionHelper(const void* opt_address, case OptionType::kDouble: *value = std::to_string(*(static_cast(opt_address))); break; + case OptionType::kAtomicInt: + *value = std::to_string(static_cast*>(opt_address) + ->load(std::memory_order_acquire)); + break; case OptionType::kString: *value = EscapeOptionString(*(static_cast(opt_address))); @@ -896,7 +914,7 @@ Status OptionTypeInfo::Parse(const ConfigOptions& config_options, ConfigOptions copy = config_options; copy.ignore_unknown_options = false; copy.invoke_prepare_options = false; - if (opt_value.find("=") != std::string::npos) { + if (opt_value.find('=') != std::string::npos) { return config->ConfigureFromString(copy, opt_value); } else { return config->ConfigureOption(copy, opt_name, opt_value); @@ -1031,7 +1049,7 @@ Status OptionTypeInfo::Serialize(const ConfigOptions& config_options, } std::string value = custom->ToString(embedded); if (!embedded.mutable_options_only || - value.find("=") != std::string::npos) { + value.find('=') != std::string::npos) { *opt_value = value; } else { *opt_value = ""; @@ -1163,6 +1181,8 @@ static bool AreOptionsEqual(OptionType type, const void* this_offset, GetUnaligned(static_cast(that_offset), &v2); return (v1 == v2); } + case OptionType::kAtomicInt: + return IsOptionEqual>(this_offset, that_offset); case OptionType::kString: return IsOptionEqual(this_offset, that_offset); case OptionType::kDouble: @@ -1405,7 +1425,7 @@ const OptionTypeInfo* OptionTypeInfo::Find( *elem_name = opt_name; // Return the name return &(iter->second); // Return the contents of the iterator } else { - auto idx = opt_name.find("."); // Look for a separator + auto idx = opt_name.find('.'); // Look for a separator if (idx > 0 && idx != std::string::npos) { // We found a separator auto siter = opt_map.find(opt_name.substr(0, idx)); // Look for the short name diff --git a/options/options_parser.cc b/options/options_parser.cc index a8c855d6e..e2431016d 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -179,8 +179,8 @@ Status RocksDBOptionsParser::ParseSection(OptionSection* section, *section = kOptionSectionUnknown; // A section is of the form [ ""], where // "" is optional. - size_t arg_start_pos = line.find("\""); - size_t arg_end_pos = line.rfind("\""); + size_t arg_start_pos = line.find('\"'); + size_t arg_end_pos = line.rfind('\"'); // The following if-then check tries to identify whether the input // section has the optional section argument. if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) { @@ -224,7 +224,7 @@ Status RocksDBOptionsParser::ParseStatement(std::string* name, std::string* value, const std::string& line, const int line_num) { - size_t eq_pos = line.find("="); + size_t eq_pos = line.find('='); if (eq_pos == std::string::npos) { return InvalidArgument(line_num, "A valid statement must have a '='."); } diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index 19cb6310f..2f7493f32 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -252,6 +252,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { sizeof(FileTypeSet)}, {offsetof(struct DBOptions, compaction_service), sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, daily_offpeak_time_utc), sizeof(std::string)}, }; char* options_ptr = new char[sizeof(DBOptions)]; @@ -308,6 +309,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { "writable_file_max_buffer_size=1048576;" "paranoid_checks=true;" "flush_verify_memtable_count=true;" + "compaction_verify_record_count=true;" "track_and_verify_wals_in_manifest=true;" "verify_sst_unique_id_in_manifest=true;" "is_fd_close_on_exec=false;" @@ -364,7 +366,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { "db_host_id=hostname;" "lowest_used_cache_tier=kNonVolatileBlockTier;" "allow_data_in_errors=false;" - "enforce_single_del_contracts=false;", + "enforce_single_del_contracts=false;" + "daily_offpeak_time_utc=08:30-19:00;", new_options)); ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), @@ -377,6 +380,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { delete[] new_options_ptr; } +// status check adds CXX flag -fno-elide-constructors which fails this test. +#ifndef ROCKSDB_ASSERT_STATUS_CHECKED // If the test fails, likely a new option is added to ColumnFamilyOptions // but it cannot be set through GetColumnFamilyOptionsFromString(), or the // test is not updated accordingly. @@ -501,11 +506,12 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { "compression=kNoCompression;" "compression_opts={max_dict_buffer_bytes=5;use_zstd_dict_trainer=true;" "enabled=false;parallel_threads=6;zstd_max_train_bytes=7;strategy=8;max_" - "dict_bytes=9;level=10;window_bits=11;max_compressed_bytes_per_kb=987;};" + "dict_bytes=9;level=10;window_bits=11;max_compressed_bytes_per_kb=987;" + "checksum=true};" "bottommost_compression_opts={max_dict_buffer_bytes=4;use_zstd_dict_" "trainer=true;enabled=true;parallel_threads=5;zstd_max_train_bytes=6;" "strategy=7;max_dict_bytes=8;level=9;window_bits=10;max_compressed_bytes_" - "per_kb=876;};" + "per_kb=876;checksum=true};" "bottommost_compression=kDisableCompressionOption;" "level0_stop_writes_trigger=33;" "num_levels=99;" @@ -548,6 +554,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { "prepopulate_blob_cache=kDisable;" "bottommost_temperature=kWarm;" "last_level_temperature=kWarm;" + "default_temperature=kHot;" "preclude_last_level_data_seconds=86400;" "preserve_internal_time_seconds=86400;" "compaction_options_fifo={max_table_files_size=3;allow_" @@ -556,7 +563,9 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { "blob_cache=1M;" "memtable_protection_bytes_per_key=2;" "persist_user_defined_timestamps=true;" - "block_protection_bytes_per_key=1;", + "block_protection_bytes_per_key=1;" + "memtable_max_range_deletions=999999;" + "bottommost_file_compaction_delay=7200;", new_options)); ASSERT_NE(new_options->blob_cache.get(), nullptr); @@ -639,6 +648,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { delete[] mcfo2_ptr; delete[] cfo_clean_ptr; } +#endif // !ROCKSDB_ASSERT_STATUS_CHECKED #endif // !ROCKSDB_UBSAN_RUN #endif // !__clang__ #endif // OS_LINUX || OS_WIN diff --git a/options/options_test.cc b/options/options_test.cc index ef0b50843..6420ebf46 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -49,7 +49,7 @@ class OptionsTest : public testing::Test {}; class UnregisteredTableFactory : public TableFactory { public: - UnregisteredTableFactory() {} + UnregisteredTableFactory() = default; const char* Name() const override { return "Unregistered"; } using TableFactory::NewTableReader; Status NewTableReader(const ReadOptions&, const TableReaderOptions&, @@ -130,7 +130,9 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"blob_file_starting_level", "1"}, {"prepopulate_blob_cache", "kDisable"}, {"last_level_temperature", "kWarm"}, + {"default_temperature", "kHot"}, {"persist_user_defined_timestamps", "true"}, + {"memtable_max_range_deletions", "0"}, }; std::unordered_map db_options_map = { @@ -176,6 +178,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"wal_bytes_per_sync", "48"}, {"strict_bytes_per_sync", "true"}, {"preserve_deletes", "false"}, + {"daily_offpeak_time_utc", ""}, }; ColumnFamilyOptions base_cf_opt; @@ -283,7 +286,9 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable); ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm); ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm); + ASSERT_EQ(new_cf_opt.default_temperature, Temperature::kHot); ASSERT_EQ(new_cf_opt.persist_user_defined_timestamps, true); + ASSERT_EQ(new_cf_opt.memtable_max_range_deletions, 0); cf_options_map["write_buffer_size"] = "hello"; ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, @@ -354,6 +359,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast(47)); ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast(48)); ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true); + ASSERT_EQ(new_db_opt.daily_offpeak_time_utc, ""); db_options_map["max_open_files"] = "hello"; Status s = @@ -875,6 +881,7 @@ TEST_F(OptionsTest, OldInterfaceTest) { {"track_and_verify_wals_in_manifest", "true"}, {"verify_sst_unique_id_in_manifest", "true"}, {"max_open_files", "32"}, + {"daily_offpeak_time_utc", "06:30-23:30"}, }; ConfigOptions db_config_options(base_db_opt); @@ -905,11 +912,13 @@ TEST_F(OptionsTest, OldInterfaceTest) { db_config_options.ignore_unknown_options = false; ASSERT_OK(GetDBOptionsFromString( db_config_options, base_db_opt, - "create_if_missing=false;error_if_exists=false;max_open_files=42;", + "create_if_missing=false;error_if_exists=false;max_open_files=42;" + "daily_offpeak_time_utc=08:30-19:00;", &new_db_opt)); ASSERT_EQ(new_db_opt.create_if_missing, false); ASSERT_EQ(new_db_opt.error_if_exists, false); ASSERT_EQ(new_db_opt.max_open_files, 42); + ASSERT_EQ(new_db_opt.daily_offpeak_time_utc, "08:30-19:00"); s = GetDBOptionsFromString( db_config_options, base_db_opt, "create_if_missing=false;error_if_exists=false;max_open_files=42;" @@ -1579,6 +1588,7 @@ TEST_F(OptionsTest, GetMutableCFOptions) { TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { Options options; ColumnFamilyOptions base_opt, new_opt; + base_opt.comparator = test::BytewiseComparatorWithU64TsWrapper(); Random rnd(302); ConfigOptions config_options; config_options.input_strings_escaped = false; @@ -1599,6 +1609,7 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { base_options_file_content, &new_opt)); ASSERT_OK( RocksDBOptionsParser::VerifyCFOptions(config_options, base_opt, new_opt)); + ASSERT_EQ(base_opt.comparator, new_opt.comparator); if (base_opt.compaction_filter) { delete base_opt.compaction_filter; } @@ -1877,7 +1888,7 @@ TEST_F(OptionsTest, StringToMapRandomTest) { "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}", "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"}; - for (std::string base : bases) { + for (const std::string& base : bases) { for (int rand_seed = 301; rand_seed < 401; rand_seed++) { Random rnd(rand_seed); for (int attempt = 0; attempt < 10; attempt++) { @@ -1898,7 +1909,7 @@ TEST_F(OptionsTest, StringToMapRandomTest) { for (int rand_seed = 301; rand_seed < 1301; rand_seed++) { Random rnd(rand_seed); int len = rnd.Uniform(30); - std::string str = ""; + std::string str; for (int attempt = 0; attempt < len; attempt++) { // Add a random character size_t pos = static_cast( @@ -2337,7 +2348,9 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { {"blob_file_starting_level", "1"}, {"prepopulate_blob_cache", "kDisable"}, {"last_level_temperature", "kWarm"}, + {"default_temperature", "kHot"}, {"persist_user_defined_timestamps", "true"}, + {"memtable_max_range_deletions", "0"}, }; std::unordered_map db_options_map = { @@ -2488,7 +2501,9 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable); ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm); ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm); + ASSERT_EQ(new_cf_opt.default_temperature, Temperature::kHot); ASSERT_EQ(new_cf_opt.persist_user_defined_timestamps, true); + ASSERT_EQ(new_cf_opt.memtable_max_range_deletions, 0); cf_options_map["write_buffer_size"] = "hello"; ASSERT_NOK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, @@ -3539,7 +3554,7 @@ TEST_F(OptionsParserTest, ParseVersion) { "3..2", ".", ".1.2", // must have at least one digit before each dot "1.2.", "1.", "2.34."}; // must have at least one digit after each dot - for (auto iv : invalid_versions) { + for (const auto& iv : invalid_versions) { snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str()); parser.Reset(); @@ -3549,7 +3564,7 @@ TEST_F(OptionsParserTest, ParseVersion) { const std::vector valid_versions = { "1.232", "100", "3.12", "1", "12.3 ", " 1.25 "}; - for (auto vv : valid_versions) { + for (const auto& vv : valid_versions) { snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str()); parser.Reset(); ASSERT_OK(fs_->WriteToNewFile(vv, buffer)); @@ -4628,42 +4643,42 @@ TEST_F(OptionTypeInfoTest, TestCustomEnum) { TEST_F(OptionTypeInfoTest, TestBuiltinEnum) { ConfigOptions config_options; - for (auto iter : OptionsHelper::compaction_style_string_map) { + for (const auto& iter : OptionsHelper::compaction_style_string_map) { CompactionStyle e1, e2; TestParseAndCompareOption(config_options, OptionTypeInfo(0, OptionType::kCompactionStyle), "CompactionStyle", iter.first, &e1, &e2); ASSERT_EQ(e1, iter.second); } - for (auto iter : OptionsHelper::compaction_pri_string_map) { + for (const auto& iter : OptionsHelper::compaction_pri_string_map) { CompactionPri e1, e2; TestParseAndCompareOption(config_options, OptionTypeInfo(0, OptionType::kCompactionPri), "CompactionPri", iter.first, &e1, &e2); ASSERT_EQ(e1, iter.second); } - for (auto iter : OptionsHelper::compression_type_string_map) { + for (const auto& iter : OptionsHelper::compression_type_string_map) { CompressionType e1, e2; TestParseAndCompareOption(config_options, OptionTypeInfo(0, OptionType::kCompressionType), "CompressionType", iter.first, &e1, &e2); ASSERT_EQ(e1, iter.second); } - for (auto iter : OptionsHelper::compaction_stop_style_string_map) { + for (const auto& iter : OptionsHelper::compaction_stop_style_string_map) { CompactionStopStyle e1, e2; TestParseAndCompareOption( config_options, OptionTypeInfo(0, OptionType::kCompactionStopStyle), "CompactionStopStyle", iter.first, &e1, &e2); ASSERT_EQ(e1, iter.second); } - for (auto iter : OptionsHelper::checksum_type_string_map) { + for (const auto& iter : OptionsHelper::checksum_type_string_map) { ChecksumType e1, e2; TestParseAndCompareOption(config_options, OptionTypeInfo(0, OptionType::kChecksumType), "CheckSumType", iter.first, &e1, &e2); ASSERT_EQ(e1, iter.second); } - for (auto iter : OptionsHelper::encoding_type_string_map) { + for (const auto& iter : OptionsHelper::encoding_type_string_map) { EncodingType e1, e2; TestParseAndCompareOption(config_options, OptionTypeInfo(0, OptionType::kEncodingType), diff --git a/port/mmap.h b/port/mmap.h index 7342a13f9..0f385522f 100644 --- a/port/mmap.h +++ b/port/mmap.h @@ -14,6 +14,7 @@ #endif // OS_WIN #include +#include #include "rocksdb/rocksdb_namespace.h" @@ -67,4 +68,23 @@ class MemMapping { static MemMapping AllocateAnonymous(size_t length, bool huge); }; +// Simple MemMapping wrapper that presents the memory as an array of T. +// For example, +// TypedMemMapping arr = MemMapping::AllocateLazyZeroed(num_bytes); +template +class TypedMemMapping : public MemMapping { + public: + /*implicit*/ TypedMemMapping(MemMapping&& v) noexcept + : MemMapping(std::move(v)) {} + TypedMemMapping& operator=(MemMapping&& v) noexcept { + MemMapping& base = *this; + base = std::move(v); + } + + inline T* Get() const { return static_cast(MemMapping::Get()); } + inline size_t Count() const { return MemMapping::Length() / sizeof(T); } + + inline T& operator[](size_t index) const { return Get()[index]; } +}; + } // namespace ROCKSDB_NAMESPACE diff --git a/port/port_example.h b/port/port_example.h index 794149a69..2a19ffee0 100644 --- a/port/port_example.h +++ b/port/port_example.h @@ -43,7 +43,7 @@ class Mutex { // Optionally crash if this thread does not hold this mutex. // The implementation must be fast, especially if NDEBUG is // defined. The implementation is allowed to skip all checks. - void AssertHeld(); + void AssertHeld() const; }; class CondVar { diff --git a/port/port_posix.cc b/port/port_posix.cc index 3872293b8..713fecb6a 100644 --- a/port/port_posix.cc +++ b/port/port_posix.cc @@ -11,20 +11,20 @@ #include "port/port_posix.h" -#include +#include #if defined(__i386__) || defined(__x86_64__) #include #endif -#include #include -#include -#include -#include #include #include #include +#include +#include +#include #include +#include #include #include @@ -100,7 +100,7 @@ bool Mutex::TryLock() { return ret; } -void Mutex::AssertHeld() { +void Mutex::AssertHeld() const { #ifndef NDEBUG assert(locked_); #endif diff --git a/port/port_posix.h b/port/port_posix.h index cdb256a6d..95641c0c5 100644 --- a/port/port_posix.h +++ b/port/port_posix.h @@ -109,9 +109,9 @@ class Mutex { bool TryLock(); - // this will assert if the mutex is not locked - // it does NOT verify that mutex is held by a calling thread - void AssertHeld(); + // This will fail assertion if the mutex is not locked. + // It does NOT verify that mutex is held by a calling thread. + void AssertHeld() const; // Also implement std Lockable inline void lock() { Lock(); } @@ -139,7 +139,7 @@ class RWMutex { void WriteLock(); void ReadUnlock(); void WriteUnlock(); - void AssertHeld() {} + void AssertHeld() const {} private: pthread_rwlock_t mu_; // the underlying platform mutex @@ -149,6 +149,9 @@ class CondVar { public: explicit CondVar(Mutex* mu); ~CondVar(); + + Mutex* GetMutex() const { return mu_; } + void Wait(); // Timed condition wait. Returns true if timeout occurred. bool TimedWait(uint64_t abs_time_us); diff --git a/port/stack_trace.cc b/port/stack_trace.cc index 1ccf9d804..5ce459ba8 100644 --- a/port/stack_trace.cc +++ b/port/stack_trace.cc @@ -25,12 +25,14 @@ void* SaveStack(int* /*num_frames*/, int /*first_frames_to_skip*/) { #include #include -#include -#include -#include -#include +#include #include +#include +#include +#include +#include + #ifdef OS_OPENBSD #include #include @@ -48,6 +50,9 @@ void* SaveStack(int* /*num_frames*/, int /*first_frames_to_skip*/) { #endif // GLIBC version #endif // OS_LINUX +#include +#include + #include "port/lang.h" namespace ROCKSDB_NAMESPACE { @@ -191,6 +196,11 @@ void PrintStack(int first_frames_to_skip) { char* debug_env = getenv("ROCKSDB_DEBUG"); bool debug = debug_env != nullptr && strlen(debug_env) > 0; + if (!debug && getenv("ROCKSDB_NO_STACK") != nullptr) { + // Skip stack trace + return; + } + if (lldb_stack_trace || gdb_stack_trace || debug) { // Allow ouside debugger to attach, even with Yama security restrictions #ifdef PR_SET_PTRACER_ANY @@ -247,9 +257,9 @@ void PrintStack(int first_frames_to_skip) { if (lldb_stack_trace) { fprintf(stderr, "Invoking LLDB for stack trace...\n"); - // Skip top ~8 frames here in PrintStack + // Skip top ~4 frames here in PrintStack auto bt_in_lldb = - "script -l python -- for f in lldb.thread.frames[8:]: print(f)"; + "script -l python -- for f in lldb.thread.frames[4:]: print(f)"; execlp(/*cmd in PATH*/ "lldb", /*arg0*/ "lldb", "-p", attach_pid_str, "-b", "-Q", "-o", GetLldbScriptSelectThread(attach_tid), "-o", bt_in_lldb, (char*)nullptr); @@ -306,28 +316,85 @@ void* SaveStack(int* num_frames, int first_frames_to_skip) { return callstack; } +static std::atomic g_thread_handling_stack_trace{0}; +static int g_recursion_count = 0; +static std::atomic g_at_exit_called{false}; + static void StackTraceHandler(int sig) { - // reset to default handler - signal(sig, SIG_DFL); fprintf(stderr, "Received signal %d (%s)\n", sig, strsignal(sig)); - // skip the top three signal handler related frames - PrintStack(3); + // Crude recursive mutex with no signal-unsafe system calls, to avoid + // re-entrance from multiple threads and avoid core dumping while trying + // to print the stack trace. + uint64_t tid = 0; + { + const auto ptid = pthread_self(); + // pthread_t is an opaque type + memcpy(&tid, &ptid, std::min(sizeof(tid), sizeof(ptid))); + // Essentially ensure non-zero + ++tid; + } + for (;;) { + uint64_t expected = 0; + if (g_thread_handling_stack_trace.compare_exchange_strong(expected, tid)) { + // Acquired mutex + g_recursion_count = 0; + break; + } + if (expected == tid) { + ++g_recursion_count; + fprintf(stderr, "Recursive call to stack trace handler (%d)\n", + g_recursion_count); + break; + } + // Sleep before trying again + usleep(1000); + } - // Efforts to fix or suppress TSAN warnings "signal-unsafe call inside of - // a signal" have failed, so just warn the user about them. + if (g_recursion_count > 2) { + // Give up after too many recursions + fprintf(stderr, "Too many recursive calls to stack trace handler (%d)\n", + g_recursion_count); + } else { + if (g_at_exit_called.load(std::memory_order_acquire)) { + fprintf(stderr, "In a race with process already exiting...\n"); + } + + // skip the top three signal handler related frames + PrintStack(3); + + // Efforts to fix or suppress TSAN warnings "signal-unsafe call inside of + // a signal" have failed, so just warn the user about them. #ifdef __SANITIZE_THREAD__ - fprintf(stderr, - "==> NOTE: any above warnings about \"signal-unsafe call\" are\n" - "==> ignorable, as they are expected when generating a stack\n" - "==> trace because of a signal under TSAN. Consider why the\n" - "==> signal was generated to begin with, and the stack trace\n" - "==> in the TSAN warning can be useful for that. (The stack\n" - "==> trace printed by the signal handler is likely obscured\n" - "==> by TSAN output.)\n"); + fprintf(stderr, + "==> NOTE: any above warnings about \"signal-unsafe call\" are\n" + "==> ignorable, as they are expected when generating a stack\n" + "==> trace because of a signal under TSAN. Consider why the\n" + "==> signal was generated to begin with, and the stack trace\n" + "==> in the TSAN warning can be useful for that. (The stack\n" + "==> trace printed by the signal handler is likely obscured\n" + "==> by TSAN output.)\n"); #endif + } + // reset to default handler + signal(sig, SIG_DFL); // re-signal to default handler (so we still get core dump if needed...) raise(sig); + + // release the mutex, in case this is somehow recoverable + if (g_recursion_count > 0) { + --g_recursion_count; + } else { + g_thread_handling_stack_trace.store(0, std::memory_order_release); + } +} + +static void AtExit() { + // wait for stack trace handler to finish, if needed + while (g_thread_handling_stack_trace.load(std::memory_order_acquire)) { + usleep(1000); + } + g_at_exit_called.store(true, std::memory_order_release); } void InstallStackTraceHandler() { @@ -337,6 +404,7 @@ void InstallStackTraceHandler() { signal(SIGSEGV, StackTraceHandler); signal(SIGBUS, StackTraceHandler); signal(SIGABRT, StackTraceHandler); + atexit(AtExit); // Allow ouside debugger to attach, even with Yama security restrictions. // This is needed even outside of PrintStack() so that external mechanisms // can dump stacks if they suspect that a test has hung. diff --git a/port/win/port_win.h b/port/win/port_win.h index 4d9883b63..4aa10d005 100644 --- a/port/win/port_win.h +++ b/port/win/port_win.h @@ -117,7 +117,7 @@ class Mutex { // this will assert if the mutex is not locked // it does NOT verify that mutex is held by a calling thread - void AssertHeld() { + void AssertHeld() const { #ifndef NDEBUG assert(locked_); #endif @@ -159,7 +159,7 @@ class RWMutex { void WriteUnlock() { ReleaseSRWLockExclusive(&srwLock_); } // Empty as in POSIX - void AssertHeld() {} + void AssertHeld() const {} private: SRWLOCK srwLock_; @@ -170,6 +170,9 @@ class CondVar { explicit CondVar(Mutex* mu) : mu_(mu) {} ~CondVar(); + + Mutex* GetMutex() const { return mu_; } + void Wait(); bool TimedWait(uint64_t expiration_time); void Signal(); diff --git a/src.mk b/src.mk index 1aaa0a949..1f596f12d 100644 --- a/src.mk +++ b/src.mk @@ -12,6 +12,7 @@ LIB_SOURCES = \ cache/secondary_cache.cc \ cache/secondary_cache_adapter.cc \ cache/sharded_cache.cc \ + cache/tiered_secondary_cache.cc \ db/arena_wrapped_db_iter.cc \ db/blob/blob_contents.cc \ db/blob/blob_fetcher.cc \ @@ -94,6 +95,7 @@ LIB_SOURCES = \ db/wal_manager.cc \ db/wide/wide_column_serialization.cc \ db/wide/wide_columns.cc \ + db/wide/wide_columns_helper.cc \ db/write_batch.cc \ db/write_batch_base.cc \ db/write_controller.cc \ @@ -158,6 +160,7 @@ LIB_SOURCES = \ options/configurable.cc \ options/customizable.cc \ options/db_options.cc \ + options/offpeak_time_info.cc \ options/options.cc \ options/options_helper.cc \ options/options_parser.cc \ @@ -383,6 +386,7 @@ STRESS_LIB_SOURCES = \ db_stress_tool/db_stress_stat.cc \ db_stress_tool/db_stress_test_base.cc \ db_stress_tool/db_stress_tool.cc \ + db_stress_tool/db_stress_wide_merge_operator.cc \ db_stress_tool/expected_state.cc \ db_stress_tool/expected_value.cc \ db_stress_tool/no_batched_ops_stress.cc \ @@ -401,6 +405,7 @@ TEST_LIB_SOURCES = \ FOLLY_SOURCES = \ $(FOLLY_DIR)/folly/container/detail/F14Table.cpp \ $(FOLLY_DIR)/folly/detail/Futex.cpp \ + $(FOLLY_DIR)/folly/lang/Exception.cpp \ $(FOLLY_DIR)/folly/lang/SafeAssert.cpp \ $(FOLLY_DIR)/folly/lang/ToAscii.cpp \ $(FOLLY_DIR)/folly/ScopeGuard.cpp \ @@ -436,8 +441,9 @@ BENCH_MAIN_SOURCES = \ TEST_MAIN_SOURCES = \ cache/cache_test.cc \ cache/cache_reservation_manager_test.cc \ - cache/lru_cache_test.cc \ cache/compressed_secondary_cache_test.cc \ + cache/lru_cache_test.cc \ + cache/tiered_secondary_cache_test.cc \ db/blob/blob_counting_iterator_test.cc \ db/blob/blob_file_addition_test.cc \ db/blob/blob_file_builder_test.cc \ @@ -533,6 +539,7 @@ TEST_MAIN_SOURCES = \ db/wal_manager_test.cc \ db/wide/db_wide_basic_test.cc \ db/wide/wide_column_serialization_test.cc \ + db/wide/wide_columns_helper_test.cc \ db/write_batch_test.cc \ db/write_callback_test.cc \ db/write_controller_test.cc \ @@ -642,86 +649,88 @@ MICROBENCH_SOURCES = \ microbench/db_basic_bench.cc \ JNI_NATIVE_SOURCES = \ - java/rocksjni/backupenginejni.cc \ - java/rocksjni/backup_engine_options.cc \ - java/rocksjni/checkpoint.cc \ - java/rocksjni/clock_cache.cc \ - java/rocksjni/cache.cc \ - java/rocksjni/columnfamilyhandle.cc \ - java/rocksjni/compact_range_options.cc \ - java/rocksjni/compaction_filter.cc \ - java/rocksjni/compaction_filter_factory.cc \ - java/rocksjni/compaction_filter_factory_jnicallback.cc \ - java/rocksjni/compaction_job_info.cc \ - java/rocksjni/compaction_job_stats.cc \ - java/rocksjni/compaction_options.cc \ - java/rocksjni/compaction_options_fifo.cc \ - java/rocksjni/compaction_options_universal.cc \ - java/rocksjni/comparator.cc \ - java/rocksjni/comparatorjnicallback.cc \ - java/rocksjni/compression_options.cc \ - java/rocksjni/concurrent_task_limiter.cc \ - java/rocksjni/config_options.cc \ - java/rocksjni/export_import_files_metadatajni.cc \ - java/rocksjni/env.cc \ - java/rocksjni/env_flink.cc \ - java/rocksjni/env_flink_test_suite.cc \ - java/rocksjni/env_options.cc \ - java/rocksjni/event_listener.cc \ - java/rocksjni/event_listener_jnicallback.cc \ - java/rocksjni/flink_compactionfilterjni.cc \ - java/rocksjni/import_column_family_options.cc \ - java/rocksjni/ingest_external_file_options.cc \ - java/rocksjni/filter.cc \ - java/rocksjni/iterator.cc \ - java/rocksjni/jnicallback.cc \ - java/rocksjni/loggerjnicallback.cc \ - java/rocksjni/lru_cache.cc \ - java/rocksjni/memtablejni.cc \ - java/rocksjni/memory_util.cc \ - java/rocksjni/merge_operator.cc \ - java/rocksjni/native_comparator_wrapper_test.cc \ - java/rocksjni/optimistic_transaction_db.cc \ - java/rocksjni/optimistic_transaction_options.cc \ - java/rocksjni/options.cc \ - java/rocksjni/options_util.cc \ - java/rocksjni/persistent_cache.cc \ - java/rocksjni/ratelimiterjni.cc \ - java/rocksjni/remove_emptyvalue_compactionfilterjni.cc \ - java/rocksjni/cassandra_compactionfilterjni.cc \ - java/rocksjni/cassandra_value_operator.cc \ - java/rocksjni/restorejni.cc \ - java/rocksjni/rocks_callback_object.cc \ - java/rocksjni/rocksjni.cc \ - java/rocksjni/rocksdb_exception_test.cc \ - java/rocksjni/slice.cc \ - java/rocksjni/snapshot.cc \ - java/rocksjni/sst_file_manager.cc \ - java/rocksjni/sst_file_writerjni.cc \ - java/rocksjni/sst_file_readerjni.cc \ - java/rocksjni/sst_file_reader_iterator.cc \ - java/rocksjni/sst_partitioner.cc \ - java/rocksjni/statistics.cc \ - java/rocksjni/statisticsjni.cc \ - java/rocksjni/table.cc \ - java/rocksjni/table_filter.cc \ - java/rocksjni/table_filter_jnicallback.cc \ - java/rocksjni/thread_status.cc \ - java/rocksjni/trace_writer.cc \ - java/rocksjni/trace_writer_jnicallback.cc \ - java/rocksjni/transaction.cc \ - java/rocksjni/transaction_db.cc \ - java/rocksjni/transaction_options.cc \ - java/rocksjni/transaction_db_options.cc \ - java/rocksjni/transaction_log.cc \ - java/rocksjni/transaction_notifier.cc \ - java/rocksjni/transaction_notifier_jnicallback.cc \ - java/rocksjni/ttl.cc \ - java/rocksjni/testable_event_listener.cc \ - java/rocksjni/wal_filter.cc \ - java/rocksjni/wal_filter_jnicallback.cc \ - java/rocksjni/write_batch.cc \ - java/rocksjni/writebatchhandlerjnicallback.cc \ - java/rocksjni/write_batch_test.cc \ - java/rocksjni/write_batch_with_index.cc \ - java/rocksjni/write_buffer_manager.cc + java/forstjni/backupenginejni.cc \ + java/forstjni/backup_engine_options.cc \ + java/forstjni/checkpoint.cc \ + java/forstjni/clock_cache.cc \ + java/forstjni/cache.cc \ + java/forstjni/columnfamilyhandle.cc \ + java/forstjni/compact_range_options.cc \ + java/forstjni/compaction_filter.cc \ + java/forstjni/compaction_filter_factory.cc \ + java/forstjni/compaction_filter_factory_jnicallback.cc \ + java/forstjni/compaction_job_info.cc \ + java/forstjni/compaction_job_stats.cc \ + java/forstjni/compaction_options.cc \ + java/forstjni/compaction_options_fifo.cc \ + java/forstjni/compaction_options_universal.cc \ + java/forstjni/comparator.cc \ + java/forstjni/comparatorjnicallback.cc \ + java/forstjni/compression_options.cc \ + java/forstjni/concurrent_task_limiter.cc \ + java/forstjni/config_options.cc \ + java/forstjni/export_import_files_metadatajni.cc \ + java/forstjni/env.cc \ + java/forstjni/env_flink.cc \ + java/forstjni/env_flink_test_suite.cc \ + java/forstjni/env_options.cc \ + java/forstjni/event_listener.cc \ + java/forstjni/event_listener_jnicallback.cc \ + java/forstjni/import_column_family_options.cc \ + java/forstjni/flink_compactionfilterjni.cc \ + java/forstjni/ingest_external_file_options.cc \ + java/forstjni/filter.cc \ + java/forstjni/hyper_clock_cache.cc \ + java/forstjni/iterator.cc \ + java/forstjni/jni_perf_context.cc \ + java/forstjni/jnicallback.cc \ + java/forstjni/loggerjnicallback.cc \ + java/forstjni/lru_cache.cc \ + java/forstjni/memtablejni.cc \ + java/forstjni/memory_util.cc \ + java/forstjni/merge_operator.cc \ + java/forstjni/native_comparator_wrapper_test.cc \ + java/forstjni/optimistic_transaction_db.cc \ + java/forstjni/optimistic_transaction_options.cc \ + java/forstjni/options.cc \ + java/forstjni/options_util.cc \ + java/forstjni/persistent_cache.cc \ + java/forstjni/ratelimiterjni.cc \ + java/forstjni/remove_emptyvalue_compactionfilterjni.cc \ + java/forstjni/cassandra_compactionfilterjni.cc \ + java/forstjni/cassandra_value_operator.cc \ + java/forstjni/restorejni.cc \ + java/forstjni/rocks_callback_object.cc \ + java/forstjni/rocksjni.cc \ + java/forstjni/rocksdb_exception_test.cc \ + java/forstjni/slice.cc \ + java/forstjni/snapshot.cc \ + java/forstjni/sst_file_manager.cc \ + java/forstjni/sst_file_writerjni.cc \ + java/forstjni/sst_file_readerjni.cc \ + java/forstjni/sst_file_reader_iterator.cc \ + java/forstjni/sst_partitioner.cc \ + java/forstjni/statistics.cc \ + java/forstjni/statisticsjni.cc \ + java/forstjni/table.cc \ + java/forstjni/table_filter.cc \ + java/forstjni/table_filter_jnicallback.cc \ + java/forstjni/thread_status.cc \ + java/forstjni/trace_writer.cc \ + java/forstjni/trace_writer_jnicallback.cc \ + java/forstjni/transaction.cc \ + java/forstjni/transaction_db.cc \ + java/forstjni/transaction_options.cc \ + java/forstjni/transaction_db_options.cc \ + java/forstjni/transaction_log.cc \ + java/forstjni/transaction_notifier.cc \ + java/forstjni/transaction_notifier_jnicallback.cc \ + java/forstjni/ttl.cc \ + java/forstjni/testable_event_listener.cc \ + java/forstjni/wal_filter.cc \ + java/forstjni/wal_filter_jnicallback.cc \ + java/forstjni/write_batch.cc \ + java/forstjni/writebatchhandlerjnicallback.cc \ + java/forstjni/write_batch_test.cc \ + java/forstjni/write_batch_with_index.cc \ + java/forstjni/write_buffer_manager.cc diff --git a/table/block_based/block.cc b/table/block_based/block.cc index 13e3397a1..bc18dd926 100644 --- a/table/block_based/block.cc +++ b/table/block_based/block.cc @@ -49,8 +49,12 @@ struct DecodeEntry { // Fast path: all three values are encoded in one byte each p += 3; } else { - if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr; - if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) { + return nullptr; + } + if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) { + return nullptr; + } if ((p = GetVarint32Ptr(p, limit, value_length)) == nullptr) { return nullptr; } @@ -82,8 +86,12 @@ struct CheckAndDecodeEntry { // Fast path: all three values are encoded in one byte each p += 3; } else { - if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr; - if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) { + return nullptr; + } + if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) { + return nullptr; + } if ((p = GetVarint32Ptr(p, limit, value_length)) == nullptr) { return nullptr; } @@ -113,15 +121,21 @@ struct DecodeKeyV4 { // We need 2 bytes for shared and non_shared size. We also need one more // byte either for value size or the actual value in case of value delta // encoding. - if (limit - p < 3) return nullptr; + if (limit - p < 3) { + return nullptr; + } *shared = reinterpret_cast(p)[0]; *non_shared = reinterpret_cast(p)[1]; if ((*shared | *non_shared) < 128) { // Fast path: all three values are encoded in one byte each p += 2; } else { - if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr; - if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) { + return nullptr; + } + if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) { + return nullptr; + } } return p; } @@ -140,7 +154,9 @@ struct DecodeEntryV4 { void DataBlockIter::NextImpl() { #ifndef NDEBUG - if (TEST_Corrupt_Callback("DataBlockIter::NextImpl")) return; + if (TEST_Corrupt_Callback("DataBlockIter::NextImpl")) { + return; + } #endif bool is_shared = false; ParseNextDataKey(&is_shared); @@ -446,7 +462,9 @@ bool DataBlockIter::SeekForGetImpl(const Slice& target) { void IndexBlockIter::SeekImpl(const Slice& target) { #ifndef NDEBUG - if (TEST_Corrupt_Callback("IndexBlockIter::SeekImpl")) return; + if (TEST_Corrupt_Callback("IndexBlockIter::SeekImpl")) { + return; + } #endif TEST_SYNC_POINT("IndexBlockIter::Seek:0"); PERF_TIMER_GUARD(block_seek_nanos); @@ -560,7 +578,9 @@ void MetaBlockIter::SeekToFirstImpl() { void IndexBlockIter::SeekToFirstImpl() { #ifndef NDEBUG - if (TEST_Corrupt_Callback("IndexBlockIter::SeekToFirstImpl")) return; + if (TEST_Corrupt_Callback("IndexBlockIter::SeekToFirstImpl")) { + return; + } #endif if (data_ == nullptr) { // Not init yet return; @@ -671,7 +691,8 @@ bool DataBlockIter::ParseNextDataKey(bool* is_shared) { // If we are reading a file with a global sequence number we should // expect that all encoded sequence numbers are zeros and any value // type is kTypeValue, kTypeMerge, kTypeDeletion, - // kTypeDeletionWithTimestamp, or kTypeRangeDeletion. + // kTypeDeletionWithTimestamp, kTypeRangeDeletion, or + // kTypeWideColumnEntity. uint64_t packed = ExtractInternalKeyFooter(raw_key_.GetKey()); SequenceNumber seqno; ValueType value_type; @@ -680,7 +701,8 @@ bool DataBlockIter::ParseNextDataKey(bool* is_shared) { value_type == ValueType::kTypeMerge || value_type == ValueType::kTypeDeletion || value_type == ValueType::kTypeDeletionWithTimestamp || - value_type == ValueType::kTypeRangeDeletion); + value_type == ValueType::kTypeRangeDeletion || + value_type == ValueType::kTypeWideColumnEntity); assert(seqno == 0); } #endif // NDEBUG @@ -736,7 +758,8 @@ void IndexBlockIter::DecodeCurrentValue(bool is_shared) { assert(value_type == ValueType::kTypeValue || value_type == ValueType::kTypeMerge || value_type == ValueType::kTypeDeletion || - value_type == ValueType::kTypeRangeDeletion); + value_type == ValueType::kTypeRangeDeletion || + value_type == ValueType::kTypeWideColumnEntity); first_internal_key.UpdateInternalKey(global_seqno_state_->global_seqno, value_type); @@ -907,7 +930,9 @@ bool IndexBlockIter::BinaryBlockIndexSeek(const Slice& target, // Key at "target" is <= "mid". Therefore all blocks // after "mid" are uninteresting. // If there is only one block left, we found it. - if (left == right) break; + if (left == right) { + break; + } right = mid; } } diff --git a/table/block_based/block_based_table_builder.cc b/table/block_based/block_based_table_builder.cc index d3c70536d..7b8bd0275 100644 --- a/table/block_based/block_based_table_builder.cc +++ b/table/block_based/block_based_table_builder.cc @@ -9,10 +9,9 @@ #include "table/block_based/block_based_table_builder.h" -#include -#include - #include +#include +#include #include #include #include @@ -138,8 +137,8 @@ Slice CompressBlock(const Slice& uncompressed_data, const CompressionInfo& info, if (sampled_output_fast && (LZ4_Supported() || Snappy_Supported())) { CompressionType c = LZ4_Supported() ? kLZ4Compression : kSnappyCompression; - CompressionContext context(c); CompressionOptions options; + CompressionContext context(c, options); CompressionInfo info_tmp(options, context, CompressionDict::GetEmptyDict(), c, info.SampleForCompression()); @@ -152,8 +151,8 @@ Slice CompressBlock(const Slice& uncompressed_data, const CompressionInfo& info, // Sampling with a slow but high-compression algorithm if (sampled_output_slow && (ZSTD_Supported() || Zlib_Supported())) { CompressionType c = ZSTD_Supported() ? kZSTD : kZlibCompression; - CompressionContext context(c); CompressionOptions options; + CompressionContext context(c, options); CompressionInfo info_tmp(options, context, CompressionDict::GetEmptyDict(), c, info.SampleForCompression()); @@ -231,7 +230,6 @@ class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector uint64_t /* block_compressed_bytes_slow */) override { // Intentionally left blank. No interest in collecting stats for // blocks. - return; } Status Finish(UserCollectedProperties* properties) override { @@ -263,7 +261,9 @@ class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector struct BlockBasedTableBuilder::Rep { const ImmutableOptions ioptions; - const MutableCFOptions moptions; + // BEGIN from MutableCFOptions + std::shared_ptr prefix_extractor; + // END from MutableCFOptions const BlockBasedTableOptions table_options; const InternalKeyComparator& internal_comparator; // Size in bytes for the user-defined timestamps. @@ -361,6 +361,9 @@ struct BlockBasedTableBuilder::Rep { // all blocks after data blocks till the end of the SST file. uint64_t tail_size; + // See class Footer + uint32_t base_context_checksum; + uint64_t get_offset() { return offset.load(std::memory_order_relaxed); } void set_offset(uint64_t o) { offset.store(o, std::memory_order_relaxed); } @@ -389,6 +392,12 @@ struct BlockBasedTableBuilder::Rep { // to false, and this is ensured by io_status_mutex, so no special memory // order for io_status_ok is required. if (io_status_ok.load(std::memory_order_relaxed)) { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED // Avoid unnecessary lock acquisition + auto ios = CopyIOStatus(); + ios.PermitUncheckedError(); + // Assume no races in unit tests + assert(ios.ok()); +#endif // ROCKSDB_ASSERT_STATUS_CHECKED return IOStatus::OK(); } else { return CopyIOStatus(); @@ -429,7 +438,7 @@ struct BlockBasedTableBuilder::Rep { Rep(const BlockBasedTableOptions& table_opt, const TableBuilderOptions& tbo, WritableFileWriter* f) : ioptions(tbo.ioptions), - moptions(tbo.moptions), + prefix_extractor(tbo.moptions.prefix_extractor), table_options(table_opt), internal_comparator(tbo.internal_comparator), ts_sz(tbo.internal_comparator.user_comparator()->timestamp_size()), @@ -456,7 +465,7 @@ struct BlockBasedTableBuilder::Rep { BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */, 0.75 /* data_block_hash_table_util_ratio */, ts_sz, persist_user_defined_timestamps), - internal_prefix_transform(tbo.moptions.prefix_extractor.get()), + internal_prefix_transform(prefix_extractor.get()), compression_type(tbo.compression_type), sample_for_compression(tbo.moptions.sample_for_compression), compressible_input_data_bytes(0), @@ -477,7 +486,7 @@ struct BlockBasedTableBuilder::Rep { flush_block_policy( table_options.flush_block_policy_factory->NewFlushBlockPolicy( table_options, data_block)), - create_context(&table_options, ioptions.stats, + create_context(&table_options, &ioptions, ioptions.stats, compression_type == kZSTD || compression_type == kZSTDNotFinalCompression, tbo.moptions.block_protection_bytes_per_key, @@ -514,8 +523,10 @@ struct BlockBasedTableBuilder::Rep { compression_dict_buffer_cache_res_mgr = nullptr; } + assert(compression_ctxs.size() >= compression_opts.parallel_threads); for (uint32_t i = 0; i < compression_opts.parallel_threads; i++) { - compression_ctxs[i].reset(new CompressionContext(compression_type)); + compression_ctxs[i].reset( + new CompressionContext(compression_type, compression_opts)); } if (table_options.index_type == BlockBasedTableOptions::kTwoLevelIndexSearch) { @@ -557,7 +568,7 @@ struct BlockBasedTableBuilder::Rep { } filter_builder.reset(CreateFilterBlockBuilder( - ioptions, moptions, filter_context, + ioptions, tbo.moptions, filter_context, use_delta_encoding_for_index_values, p_index_builder_, ts_sz, persist_user_defined_timestamps)); } @@ -566,14 +577,17 @@ struct BlockBasedTableBuilder::Rep { for (auto& factory : *tbo.int_tbl_prop_collector_factories) { assert(factory); - table_properties_collectors.emplace_back( + std::unique_ptr collector{ factory->CreateIntTblPropCollector(tbo.column_family_id, - tbo.level_at_creation)); + tbo.level_at_creation)}; + if (collector) { + table_properties_collectors.emplace_back(std::move(collector)); + } } table_properties_collectors.emplace_back( new BlockBasedTablePropertiesCollector( table_options.index_type, table_options.whole_key_filtering, - moptions.prefix_extractor != nullptr)); + prefix_extractor != nullptr)); if (ts_sz > 0 && persist_user_defined_timestamps) { table_properties_collectors.emplace_back( new TimestampTablePropertiesCollector( @@ -597,6 +611,17 @@ struct BlockBasedTableBuilder::Rep { if (!ReifyDbHostIdProperty(ioptions.env, &props.db_host_id).ok()) { ROCKS_LOG_INFO(ioptions.logger, "db_host_id property will not be set"); } + + if (FormatVersionUsesContextChecksum(table_options.format_version)) { + // Must be non-zero and semi- or quasi-random + // TODO: ideally guaranteed different for related files (e.g. use file + // number and db_session, for benefit of SstFileWriter) + do { + base_context_checksum = Random::GetTLSInstance()->Next(); + } while (UNLIKELY(base_context_checksum == 0)); + } else { + base_context_checksum = 0; + } } Rep(const Rep&) = delete; @@ -961,7 +986,9 @@ BlockBasedTableBuilder::~BlockBasedTableBuilder() { void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) { Rep* r = rep_; assert(rep_->state != Rep::State::kClosed); - if (!ok()) return; + if (!ok()) { + return; + } ValueType value_type = ExtractValueType(key); if (IsValueType(value_type)) { #ifndef NDEBUG @@ -1073,8 +1100,12 @@ void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) { void BlockBasedTableBuilder::Flush() { Rep* r = rep_; assert(rep_->state != Rep::State::kClosed); - if (!ok()) return; - if (r->data_block.empty()) return; + if (!ok()) { + return; + } + if (r->data_block.empty()) { + return; + } if (r->IsParallelCompressionEnabled() && r->state == Rep::State::kUnbuffered) { r->data_block.Finish(); @@ -1123,6 +1154,9 @@ void BlockBasedTableBuilder::WriteBlock(const Slice& uncompressed_block_data, return; } + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::WriteBlock:TamperWithCompressedData", + &r->compressed_output); WriteMaybeCompressedBlock(block_contents, type, handle, block_type, &uncompressed_block_data); r->compressed_output.clear(); @@ -1285,7 +1319,8 @@ void BlockBasedTableBuilder::WriteMaybeCompressedBlock( bool is_data_block = block_type == BlockType::kData; // Old, misleading name of this function: WriteRawBlock StopWatch sw(r->ioptions.clock, r->ioptions.stats, WRITE_RAW_BLOCK_MICROS); - handle->set_offset(r->get_offset()); + const uint64_t offset = r->get_offset(); + handle->set_offset(offset); handle->set_size(block_contents.size()); assert(status().ok()); assert(io_status().ok()); @@ -1307,6 +1342,7 @@ void BlockBasedTableBuilder::WriteMaybeCompressedBlock( uint32_t checksum = ComputeBuiltinChecksumWithLastByte( r->table_options.checksum, block_contents.data(), block_contents.size(), /*last_byte*/ comp_type); + checksum += ChecksumModifierForContext(r->base_context_checksum, offset); if (block_type == BlockType::kFilter) { Status s = r->filter_builder->MaybePostVerifyFilter(block_contents); @@ -1600,6 +1636,11 @@ void BlockBasedTableBuilder::WriteIndexBlock( // The last index_block_handle will be for the partition index block } } + // If success and need to record in metaindex rather than footer... + if (!FormatVersionUsesIndexHandleInFooter( + rep_->table_options.format_version)) { + meta_index_builder->Add(kIndexBlockName, *index_block_handle); + } } void BlockBasedTableBuilder::WritePropertiesBlock( @@ -1625,9 +1666,7 @@ void BlockBasedTableBuilder::WritePropertiesBlock( rep_->props.compression_options = CompressionOptionsToString(rep_->compression_opts); rep_->props.prefix_extractor_name = - rep_->moptions.prefix_extractor != nullptr - ? rep_->moptions.prefix_extractor->AsString() - : "nullptr"; + rep_->prefix_extractor ? rep_->prefix_extractor->AsString() : "nullptr"; std::string property_collectors_names = "["; for (size_t i = 0; i < rep_->ioptions.table_properties_collector_factories.size(); ++i) { @@ -1678,9 +1717,10 @@ void BlockBasedTableBuilder::WritePropertiesBlock( property_block_builder.AddTableProperty(rep_->props); // Add use collected properties - NotifyCollectTableCollectorsOnFinish(rep_->table_properties_collectors, - rep_->ioptions.logger, - &property_block_builder); + NotifyCollectTableCollectorsOnFinish( + rep_->table_properties_collectors, rep_->ioptions.logger, + &property_block_builder, rep_->props.user_collected_properties, + rep_->props.readable_properties); Slice block_data = property_block_builder.Finish(); TEST_SYNC_POINT_CALLBACK( @@ -1746,16 +1786,20 @@ void BlockBasedTableBuilder::WriteRangeDelBlock( void BlockBasedTableBuilder::WriteFooter(BlockHandle& metaindex_block_handle, BlockHandle& index_block_handle) { + assert(ok()); Rep* r = rep_; // this is guaranteed by BlockBasedTableBuilder's constructor assert(r->table_options.checksum == kCRC32c || r->table_options.format_version != 0); - assert(ok()); - FooterBuilder footer; - footer.Build(kBlockBasedTableMagicNumber, r->table_options.format_version, - r->get_offset(), r->table_options.checksum, - metaindex_block_handle, index_block_handle); + Status s = footer.Build(kBlockBasedTableMagicNumber, + r->table_options.format_version, r->get_offset(), + r->table_options.checksum, metaindex_block_handle, + index_block_handle, r->base_context_checksum); + if (!s.ok()) { + r->SetStatus(s); + return; + } IOStatus ios = r->file->Append(footer.GetSlice()); if (ios.ok()) { r->set_offset(r->get_offset() + footer.GetSlice().size()); @@ -1970,10 +2014,14 @@ Status BlockBasedTableBuilder::Finish() { WriteFooter(metaindex_block_handle, index_block_handle); } r->state = Rep::State::kClosed; - r->SetStatus(r->CopyIOStatus()); - Status ret_status = r->CopyStatus(); - assert(!ret_status.ok() || io_status().ok()); r->tail_size = r->offset - r->props.tail_start_offset; + + Status ret_status = r->CopyStatus(); + IOStatus ios = r->GetIOStatus(); + if (!ios.ok() && ret_status.ok()) { + // Let io_status supersede ok status (otherwise status takes precedennce) + ret_status = ios; + } return ret_status; } @@ -1983,8 +2031,10 @@ void BlockBasedTableBuilder::Abandon() { StopParallelCompression(); } rep_->state = Rep::State::kClosed; +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED // Avoid unnecessary lock acquisition rep_->CopyStatus().PermitUncheckedError(); rep_->CopyIOStatus().PermitUncheckedError(); +#endif // ROCKSDB_ASSERT_STATUS_CHECKED } uint64_t BlockBasedTableBuilder::NumEntries() const { @@ -2019,14 +2069,7 @@ bool BlockBasedTableBuilder::NeedCompact() const { } TableProperties BlockBasedTableBuilder::GetTableProperties() const { - TableProperties ret = rep_->props; - for (const auto& collector : rep_->table_properties_collectors) { - for (const auto& prop : collector->GetReadableProperties()) { - ret.readable_properties.insert(prop); - } - collector->Finish(&ret.user_collected_properties).PermitUncheckedError(); - } - return ret; + return rep_->props; } std::string BlockBasedTableBuilder::GetFileChecksum() const { diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index 4ef5da419..67b8f704b 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -9,9 +9,8 @@ #include "table/block_based/block_based_table_factory.h" -#include - #include +#include #include #include @@ -225,7 +224,6 @@ static std::unordered_map block_based_table_type_info = { /* currently not supported @@ -310,6 +308,9 @@ static std::unordered_map {offsetof(struct BlockBasedTableOptions, optimize_filters_for_memory), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + // TODO "use_delta_encoding" has not been persisted - + // this may have been an omission, but changing this now might be a + // breaker {"filter_policy", OptionTypeInfo::AsCustomSharedPtr( offsetof(struct BlockBasedTableOptions, filter_policy), diff --git a/table/block_based/block_based_table_iterator.cc b/table/block_based/block_based_table_iterator.cc index 57744a587..d08def5a0 100644 --- a/table/block_based/block_based_table_iterator.cc +++ b/table/block_based/block_based_table_iterator.cc @@ -16,12 +16,42 @@ void BlockBasedTableIterator::Seek(const Slice& target) { SeekImpl(&target, true); } +void BlockBasedTableIterator::SeekSecondPass(const Slice* target) { + AsyncInitDataBlock(/*is_first_pass=*/false); + + if (target) { + block_iter_.Seek(*target); + } else { + block_iter_.SeekToFirst(); + } + FindKeyForward(); + + CheckOutOfBound(); + + if (target) { + assert(!Valid() || icomp_.Compare(*target, key()) <= 0); + } +} + void BlockBasedTableIterator::SeekImpl(const Slice* target, bool async_prefetch) { - bool is_first_pass = true; - if (async_read_in_progress_) { - AsyncInitDataBlock(false); - is_first_pass = false; + bool is_first_pass = !async_read_in_progress_; + + if (!is_first_pass) { + SeekSecondPass(target); + return; + } + + ResetBlockCacheLookupVar(); + + bool autotune_readaheadsize = is_first_pass && + read_options_.auto_readahead_size && + read_options_.iterate_upper_bound; + + if (autotune_readaheadsize && + table_->get_rep()->table_options.block_cache.get() && + direction_ == IterDirection::kForward) { + readahead_cache_lookup_ = true; } is_out_of_bound_ = false; @@ -44,7 +74,12 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target, } bool need_seek_index = true; - if (block_iter_points_to_real_block_ && block_iter_.Valid()) { + + // In case of readahead_cache_lookup_, index_iter_ could change to find the + // readahead size in BlockCacheLookupForReadAheadSize so it needs to + // reseek. + if (IsIndexAtCurr() && block_iter_points_to_real_block_ && + block_iter_.Valid()) { // Reseek. prev_block_offset_ = index_iter_->value().handle.offset(); @@ -72,13 +107,15 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target, } else { index_iter_->SeekToFirst(); } - + is_index_at_curr_block_ = true; if (!index_iter_->Valid()) { ResetDataIter(); return; } } + // After reseek, index_iter_ point to the right key i.e. target in + // case of readahead_cache_lookup_. So index_iter_ can be used directly. IndexValue v = index_iter_->value(); const bool same_block = block_iter_points_to_real_block_ && v.handle.offset() == prev_block_offset_; @@ -97,14 +134,12 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target, // Need to use the data block. if (!same_block) { if (read_options_.async_io && async_prefetch) { - if (is_first_pass) { - AsyncInitDataBlock(is_first_pass); - } + AsyncInitDataBlock(/*is_first_pass=*/true); if (async_read_in_progress_) { // Status::TryAgain indicates asynchronous request for retrieval of // data blocks has been submitted. So it should return at this point - // and Seek should be called again to retrieve the requested block and - // execute the remaining code. + // and Seek should be called again to retrieve the requested block + // and execute the remaining code. return; } } else { @@ -135,6 +170,8 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target, } void BlockBasedTableIterator::SeekForPrev(const Slice& target) { + direction_ = IterDirection::kBackward; + ResetBlockCacheLookupVar(); is_out_of_bound_ = false; is_at_first_key_from_index_ = false; seek_stat_state_ = kNone; @@ -171,6 +208,7 @@ void BlockBasedTableIterator::SeekForPrev(const Slice& target) { // to distinguish the two unless we read the second block. In this case, we'll // end up with reading two blocks. index_iter_->Seek(target); + is_index_at_curr_block_ = true; if (!index_iter_->Valid()) { auto seek_status = index_iter_->status(); @@ -206,15 +244,22 @@ void BlockBasedTableIterator::SeekForPrev(const Slice& target) { } void BlockBasedTableIterator::SeekToLast() { + direction_ = IterDirection::kBackward; + ResetBlockCacheLookupVar(); is_out_of_bound_ = false; is_at_first_key_from_index_ = false; seek_stat_state_ = kNone; + SavePrevIndexValue(); + index_iter_->SeekToLast(); + is_index_at_curr_block_ = true; + if (!index_iter_->Valid()) { ResetDataIter(); return; } + InitDataBlock(); block_iter_.SeekToLast(); FindKeyBackward(); @@ -243,6 +288,32 @@ bool BlockBasedTableIterator::NextAndGetResult(IterateResult* result) { } void BlockBasedTableIterator::Prev() { + if (readahead_cache_lookup_ && !IsIndexAtCurr()) { + // In case of readahead_cache_lookup_, index_iter_ has moved forward. So we + // need to reseek the index_iter_ to point to current block by using + // block_iter_'s key. + if (Valid()) { + ResetBlockCacheLookupVar(); + direction_ = IterDirection::kBackward; + Slice last_key = key(); + + index_iter_->Seek(last_key); + is_index_at_curr_block_ = true; + + // Check for IO error. + if (!index_iter_->Valid()) { + ResetDataIter(); + return; + } + } + + if (!Valid()) { + ResetDataIter(); + return; + } + } + + ResetBlockCacheLookupVar(); if (is_at_first_key_from_index_) { is_at_first_key_from_index_ = false; @@ -262,7 +333,18 @@ void BlockBasedTableIterator::Prev() { } void BlockBasedTableIterator::InitDataBlock() { - BlockHandle data_block_handle = index_iter_->value().handle; + BlockHandle data_block_handle; + bool is_in_cache = false; + bool use_block_cache_for_lookup = true; + + if (DoesContainBlockHandles()) { + data_block_handle = block_handles_.front().handle_; + is_in_cache = block_handles_.front().is_cache_hit_; + use_block_cache_for_lookup = false; + } else { + data_block_handle = index_iter_->value().handle; + } + if (!block_iter_points_to_real_block_ || data_block_handle.offset() != prev_block_offset_ || // if previous attempt of reading the block missed cache, try again @@ -270,25 +352,50 @@ void BlockBasedTableIterator::InitDataBlock() { if (block_iter_points_to_real_block_) { ResetDataIter(); } - auto* rep = table_->get_rep(); bool is_for_compaction = lookup_context_.caller == TableReaderCaller::kCompaction; - // Prefetch additional data for range scans (iterators). - // Implicit auto readahead: - // Enabled after 2 sequential IOs when ReadOptions.readahead_size == 0. - // Explicit user requested readahead: - // Enabled from the very first IO when ReadOptions.readahead_size is set. - block_prefetcher_.PrefetchIfNeeded( - rep, data_block_handle, read_options_.readahead_size, is_for_compaction, - /*no_sequential_checking=*/false, read_options_.rate_limiter_priority); - Status s; - table_->NewDataBlockIterator( - read_options_, data_block_handle, &block_iter_, BlockType::kData, - /*get_context=*/nullptr, &lookup_context_, - block_prefetcher_.prefetch_buffer(), - /*for_compaction=*/is_for_compaction, /*async_read=*/false, s); + + // Initialize Data Block From CacheableEntry. + if (is_in_cache) { + Status s; + block_iter_.Invalidate(Status::OK()); + table_->NewDataBlockIterator( + read_options_, (block_handles_.front().cachable_entry_).As(), + &block_iter_, s); + } else { + auto* rep = table_->get_rep(); + + std::function readaheadsize_cb = + nullptr; + if (readahead_cache_lookup_) { + readaheadsize_cb = std::bind( + &BlockBasedTableIterator::BlockCacheLookupForReadAheadSize, this, + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3); + } + + // Prefetch additional data for range scans (iterators). + // Implicit auto readahead: + // Enabled after 2 sequential IOs when ReadOptions.readahead_size == 0. + // Explicit user requested readahead: + // Enabled from the very first IO when ReadOptions.readahead_size is + // set. + block_prefetcher_.PrefetchIfNeeded( + rep, data_block_handle, read_options_.readahead_size, + is_for_compaction, + /*no_sequential_checking=*/false, read_options_, readaheadsize_cb); + + Status s; + table_->NewDataBlockIterator( + read_options_, data_block_handle, &block_iter_, BlockType::kData, + /*get_context=*/nullptr, &lookup_context_, + block_prefetcher_.prefetch_buffer(), + /*for_compaction=*/is_for_compaction, /*async_read=*/false, s, + use_block_cache_for_lookup); + } block_iter_points_to_real_block_ = true; + CheckDataBlockWithinUpperBound(); if (!is_for_compaction && (seek_stat_state_ & kDataBlockReadSinceLastSeek) == 0) { @@ -302,10 +409,11 @@ void BlockBasedTableIterator::InitDataBlock() { } void BlockBasedTableIterator::AsyncInitDataBlock(bool is_first_pass) { - BlockHandle data_block_handle = index_iter_->value().handle; + BlockHandle data_block_handle; bool is_for_compaction = lookup_context_.caller == TableReaderCaller::kCompaction; if (is_first_pass) { + data_block_handle = index_iter_->value().handle; if (!block_iter_points_to_real_block_ || data_block_handle.offset() != prev_block_offset_ || // if previous attempt of reading the block missed cache, try again @@ -314,6 +422,16 @@ void BlockBasedTableIterator::AsyncInitDataBlock(bool is_first_pass) { ResetDataIter(); } auto* rep = table_->get_rep(); + + std::function readaheadsize_cb = + nullptr; + if (readahead_cache_lookup_) { + readaheadsize_cb = std::bind( + &BlockBasedTableIterator::BlockCacheLookupForReadAheadSize, this, + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3); + } + // Prefetch additional data for range scans (iterators). // Implicit auto readahead: // Enabled after 2 sequential IOs when ReadOptions.readahead_size == 0. @@ -326,14 +444,15 @@ void BlockBasedTableIterator::AsyncInitDataBlock(bool is_first_pass) { block_prefetcher_.PrefetchIfNeeded( rep, data_block_handle, read_options_.readahead_size, is_for_compaction, /*no_sequential_checking=*/read_options_.async_io, - read_options_.rate_limiter_priority); + read_options_, readaheadsize_cb); Status s; table_->NewDataBlockIterator( read_options_, data_block_handle, &block_iter_, BlockType::kData, /*get_context=*/nullptr, &lookup_context_, block_prefetcher_.prefetch_buffer(), - /*for_compaction=*/is_for_compaction, /*async_read=*/true, s); + /*for_compaction=*/is_for_compaction, /*async_read=*/true, s, + /*use_block_cache_for_lookup=*/true); if (s.IsTryAgain()) { async_read_in_progress_ = true; @@ -343,12 +462,30 @@ void BlockBasedTableIterator::AsyncInitDataBlock(bool is_first_pass) { } else { // Second pass will call the Poll to get the data block which has been // requested asynchronously. + bool is_in_cache = false; + + if (DoesContainBlockHandles()) { + data_block_handle = block_handles_.front().handle_; + is_in_cache = block_handles_.front().is_cache_hit_; + } else { + data_block_handle = index_iter_->value().handle; + } + Status s; - table_->NewDataBlockIterator( - read_options_, data_block_handle, &block_iter_, BlockType::kData, - /*get_context=*/nullptr, &lookup_context_, - block_prefetcher_.prefetch_buffer(), - /*for_compaction=*/is_for_compaction, /*async_read=*/false, s); + // Initialize Data Block From CacheableEntry. + if (is_in_cache) { + block_iter_.Invalidate(Status::OK()); + table_->NewDataBlockIterator( + read_options_, (block_handles_.front().cachable_entry_).As(), + &block_iter_, s); + } else { + table_->NewDataBlockIterator( + read_options_, data_block_handle, &block_iter_, BlockType::kData, + /*get_context=*/nullptr, &lookup_context_, + block_prefetcher_.prefetch_buffer(), + /*for_compaction=*/is_for_compaction, /*async_read=*/false, s, + /*use_block_cache_for_lookup=*/false); + } } block_iter_points_to_real_block_ = true; CheckDataBlockWithinUpperBound(); @@ -379,20 +516,29 @@ bool BlockBasedTableIterator::MaterializeCurrentBlock() { block_iter_.SeekToFirst(); + // MaterializeCurrentBlock is called when block is actually read by + // calling InitDataBlock. is_at_first_key_from_index_ will be false for block + // handles placed in blockhandle. So index_ will be pointing to current block. + // After InitDataBlock, index_iter_ can point to different block if + // BlockCacheLookupForReadAheadSize is called. + Slice first_internal_key; + if (DoesContainBlockHandles()) { + first_internal_key = block_handles_.front().first_internal_key_; + } else { + first_internal_key = index_iter_->value().first_internal_key; + } + if (!block_iter_.Valid() || - icomp_.Compare(block_iter_.key(), - index_iter_->value().first_internal_key) != 0) { + icomp_.Compare(block_iter_.key(), first_internal_key) != 0) { block_iter_.Invalidate(Status::Corruption( "first key in index doesn't match first key in block")); return false; } - return true; } void BlockBasedTableIterator::FindKeyForward() { // This method's code is kept short to make it likely to be inlined. - assert(!is_out_of_bound_); assert(block_iter_points_to_real_block_); @@ -415,40 +561,72 @@ void BlockBasedTableIterator::FindBlockForward() { return; } // Whether next data block is out of upper bound, if there is one. - const bool next_block_is_out_of_bound = - read_options_.iterate_upper_bound != nullptr && + // index_iter_ can point to different block in case of + // readahead_cache_lookup_. readahead_cache_lookup_ will be handle the + // upper_bound check. + bool next_block_is_out_of_bound = + IsIndexAtCurr() && read_options_.iterate_upper_bound != nullptr && block_iter_points_to_real_block_ && block_upper_bound_check_ == BlockUpperBound::kUpperBoundInCurBlock; + assert(!next_block_is_out_of_bound || user_comparator_.CompareWithoutTimestamp( *read_options_.iterate_upper_bound, /*a_has_ts=*/false, index_iter_->user_key(), /*b_has_ts=*/true) <= 0); + ResetDataIter(); - index_iter_->Next(); - if (next_block_is_out_of_bound) { - // The next block is out of bound. No need to read it. - TEST_SYNC_POINT_CALLBACK("BlockBasedTableIterator:out_of_bound", nullptr); - // We need to make sure this is not the last data block before setting - // is_out_of_bound_, since the index key for the last data block can be - // larger than smallest key of the next file on the same level. - if (index_iter_->Valid()) { - is_out_of_bound_ = true; - } - return; - } - if (!index_iter_->Valid()) { - return; + if (DoesContainBlockHandles()) { + // Advance and point to that next Block handle to make that block handle + // current. + block_handles_.pop_front(); } - IndexValue v = index_iter_->value(); + if (!DoesContainBlockHandles()) { + // For readahead_cache_lookup_ enabled scenario - + // 1. In case of Seek, block_handle will be empty and it should be follow + // as usual doing index_iter_->Next(). + // 2. If block_handles is empty and index is not at current because of + // lookup (during Next), it should skip doing index_iter_->Next(), as + // it's already pointing to next block; + // 3. Last block could be out of bound and it won't iterate over that + // during BlockCacheLookup. We need to set for that block here. + if (IsIndexAtCurr() || is_index_out_of_bound_) { + index_iter_->Next(); + if (is_index_out_of_bound_) { + next_block_is_out_of_bound = is_index_out_of_bound_; + is_index_out_of_bound_ = false; + } + } else { + // Skip Next as index_iter_ already points to correct index when it + // iterates in BlockCacheLookupForReadAheadSize. + is_index_at_curr_block_ = true; + } - if (!v.first_internal_key.empty() && allow_unprepared_value_) { - // Index contains the first key of the block. Defer reading the block. - is_at_first_key_from_index_ = true; - return; - } + if (next_block_is_out_of_bound) { + // The next block is out of bound. No need to read it. + TEST_SYNC_POINT_CALLBACK("BlockBasedTableIterator:out_of_bound", + nullptr); + // We need to make sure this is not the last data block before setting + // is_out_of_bound_, since the index key for the last data block can be + // larger than smallest key of the next file on the same level. + if (index_iter_->Valid()) { + is_out_of_bound_ = true; + } + return; + } + + if (!index_iter_->Valid()) { + return; + } + IndexValue v = index_iter_->value(); + if (!v.first_internal_key.empty() && allow_unprepared_value_) { + // Index contains the first key of the block. Defer reading the block. + is_at_first_key_from_index_ = true; + return; + } + } InitDataBlock(); block_iter_.SeekToFirst(); } while (!block_iter_.Valid()); @@ -487,7 +665,7 @@ void BlockBasedTableIterator::CheckOutOfBound() { } void BlockBasedTableIterator::CheckDataBlockWithinUpperBound() { - if (read_options_.iterate_upper_bound != nullptr && + if (IsIndexAtCurr() && read_options_.iterate_upper_bound != nullptr && block_iter_points_to_real_block_) { block_upper_bound_check_ = (user_comparator_.CompareWithoutTimestamp( *read_options_.iterate_upper_bound, @@ -497,4 +675,199 @@ void BlockBasedTableIterator::CheckDataBlockWithinUpperBound() { : BlockUpperBound::kUpperBoundInCurBlock; } } + +void BlockBasedTableIterator::InitializeStartAndEndOffsets( + bool read_curr_block, bool& found_first_miss_block, + uint64_t& start_updated_offset, uint64_t& end_updated_offset, + size_t& prev_handles_size) { + prev_handles_size = block_handles_.size(); + size_t footer = table_->get_rep()->footer.GetBlockTrailerSize(); + + // It initialize start and end offset to begin which is covered by following + // scenarios + if (read_curr_block) { + if (!DoesContainBlockHandles()) { + // Scenario 1 : read_curr_block (callback made on miss block which caller + // was reading) and it has no existing handles in queue. i.e. + // index_iter_ is pointing to block that is being read by + // caller. + // + // Add current block here as it doesn't need any lookup. + BlockHandleInfo block_handle_info; + block_handle_info.handle_ = index_iter_->value().handle; + block_handle_info.SetFirstInternalKey( + index_iter_->value().first_internal_key); + + end_updated_offset = block_handle_info.handle_.offset() + footer + + block_handle_info.handle_.size(); + block_handles_.emplace_back(std::move(block_handle_info)); + + index_iter_->Next(); + is_index_at_curr_block_ = false; + found_first_miss_block = true; + } else { + // Scenario 2 : read_curr_block (callback made on miss block which caller + // was reading) but the queue already has some handles. + // + // It can be due to reading error in second buffer in FilePrefetchBuffer. + // BlockHandles already added to the queue but there was error in fetching + // those data blocks. So in this call they need to be read again. + assert(block_handles_.front().is_cache_hit_ == false); + found_first_miss_block = true; + // Initialize prev_handles_size to 0 as all those handles need to be read + // again. + prev_handles_size = 0; + start_updated_offset = block_handles_.front().handle_.offset(); + end_updated_offset = block_handles_.back().handle_.offset() + footer + + block_handles_.back().handle_.size(); + } + } else { + // Scenario 3 : read_curr_block is false (callback made to do additional + // prefetching in buffers) and the queue already has some + // handles from first buffer. + if (DoesContainBlockHandles()) { + start_updated_offset = block_handles_.back().handle_.offset() + footer + + block_handles_.back().handle_.size(); + end_updated_offset = start_updated_offset; + } else { + // Scenario 4 : read_curr_block is false (callback made to do additional + // prefetching in buffers) but the queue has no handle + // from first buffer. + // + // It can be when Reseek is from block cache (which doesn't clear the + // buffers in FilePrefetchBuffer but clears block handles from queue) and + // reseek also lies within the buffer. So Next will get data from + // exisiting buffers untill this callback is made to prefetch additional + // data. All handles need to be added to the queue starting from + // index_iter_. + assert(index_iter_->Valid()); + start_updated_offset = index_iter_->value().handle.offset(); + end_updated_offset = start_updated_offset; + } + } +} + +// BlockCacheLookupForReadAheadSize API lookups in the block cache and tries to +// reduce the start and end offset passed. +// +// Implementation - +// This function looks into the block cache for the blocks between start_offset +// and end_offset and add all the handles in the queue. +// It then iterates from the end to find first miss block and update the end +// offset to that block. +// It also iterates from the start and find first miss block and update the +// start offset to that block. +// +// Arguments - +// start_offset : Offset from which the caller wants to read. +// end_offset : End offset till which the caller wants to read. +// read_curr_block : True if this call was due to miss in the cache and +// caller wants to read that block. +// False if current call is to prefetch additional data in +// extra buffers. +void BlockBasedTableIterator::BlockCacheLookupForReadAheadSize( + bool read_curr_block, uint64_t& start_offset, uint64_t& end_offset) { + uint64_t start_updated_offset = start_offset; + + // readahead_cache_lookup_ can be set false, if after Seek and Next + // there is SeekForPrev or any other backward operation. + if (!readahead_cache_lookup_) { + return; + } + + size_t footer = table_->get_rep()->footer.GetBlockTrailerSize(); + if (read_curr_block && !DoesContainBlockHandles() && + IsNextBlockOutOfBound()) { + end_offset = index_iter_->value().handle.offset() + footer + + index_iter_->value().handle.size(); + return; + } + + uint64_t end_updated_offset = start_updated_offset; + bool found_first_miss_block = false; + size_t prev_handles_size; + + // Initialize start and end offsets based on exisiting handles in the queue + // and read_curr_block argument passed. + InitializeStartAndEndOffsets(read_curr_block, found_first_miss_block, + start_updated_offset, end_updated_offset, + prev_handles_size); + + while (index_iter_->Valid() && !is_index_out_of_bound_) { + BlockHandle block_handle = index_iter_->value().handle; + + // Adding this data block exceeds end offset. So this data + // block won't be added. + // There can be a case where passed end offset is smaller than + // block_handle.size() + footer because of readahead_size truncated to + // upper_bound. So we prefer to read the block rather than skip it to avoid + // sync read calls in case of async_io. + if (start_updated_offset != end_updated_offset && + (end_updated_offset + block_handle.size() + footer > end_offset)) { + break; + } + + // For current data block, do the lookup in the cache. Lookup should pin the + // data block in cache. + BlockHandleInfo block_handle_info; + block_handle_info.handle_ = index_iter_->value().handle; + block_handle_info.SetFirstInternalKey( + index_iter_->value().first_internal_key); + end_updated_offset += footer + block_handle_info.handle_.size(); + + Status s = table_->LookupAndPinBlocksInCache( + read_options_, block_handle, + &(block_handle_info.cachable_entry_).As()); + if (!s.ok()) { + break; + } + + block_handle_info.is_cache_hit_ = + (block_handle_info.cachable_entry_.GetValue() || + block_handle_info.cachable_entry_.GetCacheHandle()); + + // If this is the first miss block, update start offset to this block. + if (!found_first_miss_block && !block_handle_info.is_cache_hit_) { + found_first_miss_block = true; + start_updated_offset = block_handle_info.handle_.offset(); + } + + // Add the handle to the queue. + block_handles_.emplace_back(std::move(block_handle_info)); + + // Can't figure out for current block if current block + // is out of bound. But for next block we can find that. + // If curr block's index key >= iterate_upper_bound, it + // means all the keys in next block or above are out of + // bound. + if (IsNextBlockOutOfBound()) { + is_index_out_of_bound_ = true; + break; + } + index_iter_->Next(); + is_index_at_curr_block_ = false; + }; + + if (found_first_miss_block) { + // Iterate cache hit block handles from the end till a Miss is there, to + // truncate and update the end offset till that Miss. + auto it = block_handles_.rbegin(); + auto it_end = + block_handles_.rbegin() + (block_handles_.size() - prev_handles_size); + + while (it != it_end && (*it).is_cache_hit_) { + it++; + } + end_updated_offset = (*it).handle_.offset() + footer + (*it).handle_.size(); + } else { + // Nothing to read. Can be because of IOError in index_iter_->Next() or + // reached upper_bound. + end_updated_offset = start_updated_offset; + } + + end_offset = end_updated_offset; + start_offset = start_updated_offset; + ResetPreviousBlockOffset(); +} + } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_iterator.h b/table/block_based/block_based_table_iterator.h index 6ea53f331..554785305 100644 --- a/table/block_based/block_based_table_iterator.h +++ b/table/block_based/block_based_table_iterator.h @@ -7,6 +7,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #pragma once +#include + #include "table/block_based/block_based_table_reader.h" #include "table/block_based/block_based_table_reader_impl.h" #include "table/block_based/block_prefetcher.h" @@ -44,7 +46,7 @@ class BlockBasedTableIterator : public InternalIteratorBase { async_read_in_progress_(false), is_last_level_(table->IsLastLevel()) {} - ~BlockBasedTableIterator() {} + ~BlockBasedTableIterator() override { ClearBlockHandles(); } void Seek(const Slice& target) override; void SeekForPrev(const Slice& target) override; @@ -58,6 +60,11 @@ class BlockBasedTableIterator : public InternalIteratorBase { (is_at_first_key_from_index_ || (block_iter_points_to_real_block_ && block_iter_.Valid())); } + + // For block cache readahead lookup scenario - + // If is_at_first_key_from_index_ is true, InitDataBlock hasn't been + // called. It means block_handles is empty and index_ point to current block. + // So index_iter_ can be accessed directly. Slice key() const override { assert(Valid()); if (is_at_first_key_from_index_) { @@ -74,6 +81,7 @@ class BlockBasedTableIterator : public InternalIteratorBase { return block_iter_.user_key(); } } + bool PrepareValue() override { assert(Valid()); @@ -104,8 +112,12 @@ class BlockBasedTableIterator : public InternalIteratorBase { return block_iter_.value(); } Status status() const override { - // Prefix index set status to NotFound when the prefix does not exist - if (!index_iter_->status().ok() && !index_iter_->status().IsNotFound()) { + // In case of block cache readahead lookup, it won't add the block to + // block_handles if it's index is invalid. So index_iter_->status check can + // be skipped. + // Prefix index set status to NotFound when the prefix does not exist. + if (IsIndexAtCurr() && !index_iter_->status().ok() && + !index_iter_->status().IsNotFound()) { return index_iter_->status(); } else if (block_iter_points_to_real_block_) { return block_iter_.status(); @@ -159,7 +171,7 @@ class BlockBasedTableIterator : public InternalIteratorBase { } void SavePrevIndexValue() { - if (block_iter_points_to_real_block_) { + if (block_iter_points_to_real_block_ && IsIndexAtCurr()) { // Reseek. If they end up with the same data block, we shouldn't re-fetch // the same data block. prev_block_offset_ = index_iter_->value().handle.offset(); @@ -187,6 +199,10 @@ class BlockBasedTableIterator : public InternalIteratorBase { } } + FilePrefetchBuffer* prefetch_buffer() { + return block_prefetcher_.prefetch_buffer(); + } + std::unique_ptr> index_iter_; private: @@ -235,6 +251,28 @@ class BlockBasedTableIterator : public InternalIteratorBase { kReportOnUseful = 1 << 2, }; + // BlockHandleInfo is used to store the info needed when block cache lookup + // ahead is enabled to tune readahead_size. + struct BlockHandleInfo { + void SetFirstInternalKey(const Slice& key) { + if (key.empty()) { + return; + } + size_t size = key.size(); + buf_ = std::unique_ptr(new char[size]); + memcpy(buf_.get(), key.data(), size); + first_internal_key_ = Slice(buf_.get(), size); + } + + BlockHandle handle_; + bool is_cache_hit_ = false; + CachableEntry cachable_entry_; + Slice first_internal_key_; + std::unique_ptr buf_; + }; + + bool IsIndexAtCurr() const { return is_index_at_curr_block_; } + const BlockBasedTable* table_; const ReadOptions& read_options_; const InternalKeyComparator& icomp_; @@ -268,6 +306,31 @@ class BlockBasedTableIterator : public InternalIteratorBase { mutable SeekStatState seek_stat_state_ = SeekStatState::kNone; bool is_last_level_; + // If set to true, it'll lookup in the cache ahead to estimate the readahead + // size based on cache hit and miss. + bool readahead_cache_lookup_ = false; + + // It stores all the block handles that are lookuped in cache ahead when + // BlockCacheLookupForReadAheadSize is called. Since index_iter_ may point to + // different blocks when readahead_size is calculated in + // BlockCacheLookupForReadAheadSize, to avoid index_iter_ reseek, + // block_handles_ is used. + std::deque block_handles_; + + // During cache lookup to find readahead size, index_iter_ is iterated and it + // can point to a different block. is_index_at_curr_block_ keeps track of + // that. + bool is_index_at_curr_block_ = true; + bool is_index_out_of_bound_ = false; + + // Used in case of auto_readahead_size to disable the block_cache lookup if + // direction is reversed from forward to backward. In case of backward + // direction, SeekForPrev or Prev might call Seek from db_iter. So direction + // is used to disable the lookup. + IterDirection direction_ = IterDirection::kForward; + + void SeekSecondPass(const Slice* target); + // If `target` is null, seek to first. void SeekImpl(const Slice* target, bool async_prefetch); @@ -306,5 +369,47 @@ class BlockBasedTableIterator : public InternalIteratorBase { } return true; } + + // *** BEGIN APIs relevant to auto tuning of readahead_size *** + + // This API is called to lookup the data blocks ahead in the cache to tune + // the start and end offsets passed. + void BlockCacheLookupForReadAheadSize(bool read_curr_block, + uint64_t& start_offset, + uint64_t& end_offset); + + void ResetBlockCacheLookupVar() { + is_index_out_of_bound_ = false; + readahead_cache_lookup_ = false; + ClearBlockHandles(); + } + + bool IsNextBlockOutOfBound() { + // If curr block's index key >= iterate_upper_bound, it means all the keys + // in next block or above are out of bound. + return (user_comparator_.CompareWithoutTimestamp( + index_iter_->user_key(), + /*a_has_ts=*/true, *read_options_.iterate_upper_bound, + /*b_has_ts=*/false) >= 0 + ? true + : false); + } + + void ClearBlockHandles() { block_handles_.clear(); } + + // Reset prev_block_offset_. If index_iter_ has moved ahead, it won't get + // accurate prev_block_offset_. + void ResetPreviousBlockOffset() { + prev_block_offset_ = std::numeric_limits::max(); + } + + bool DoesContainBlockHandles() { return !block_handles_.empty(); } + + void InitializeStartAndEndOffsets(bool read_curr_block, + bool& found_first_miss_block, + uint64_t& start_updated_offset, + uint64_t& end_updated_offset, + size_t& prev_handles_size); + // *** END APIs relevant to auto tuning of readahead_size *** }; } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_reader.cc b/table/block_based/block_based_table_reader.cc index 35c6f46f6..a184264df 100644 --- a/table/block_based/block_based_table_reader.cc +++ b/table/block_based/block_based_table_reader.cc @@ -88,21 +88,31 @@ CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) { // Explicitly instantiate templates for each "blocklike" type we use (and // before implicit specialization). // This makes it possible to keep the template definitions in the .cc file. -#define INSTANTIATE_RETRIEVE_BLOCK(T) \ - template Status BlockBasedTable::RetrieveBlock( \ - FilePrefetchBuffer * prefetch_buffer, const ReadOptions& ro, \ - const BlockHandle& handle, const UncompressionDict& uncompression_dict, \ - CachableEntry* out_parsed_block, GetContext* get_context, \ - BlockCacheLookupContext* lookup_context, bool for_compaction, \ - bool use_cache, bool async_read) const; - -INSTANTIATE_RETRIEVE_BLOCK(ParsedFullFilterBlock); -INSTANTIATE_RETRIEVE_BLOCK(UncompressionDict); -INSTANTIATE_RETRIEVE_BLOCK(Block_kData); -INSTANTIATE_RETRIEVE_BLOCK(Block_kIndex); -INSTANTIATE_RETRIEVE_BLOCK(Block_kFilterPartitionIndex); -INSTANTIATE_RETRIEVE_BLOCK(Block_kRangeDeletion); -INSTANTIATE_RETRIEVE_BLOCK(Block_kMetaIndex); +#define INSTANTIATE_BLOCKLIKE_TEMPLATES(T) \ + template Status BlockBasedTable::RetrieveBlock( \ + FilePrefetchBuffer * prefetch_buffer, const ReadOptions& ro, \ + const BlockHandle& handle, const UncompressionDict& uncompression_dict, \ + CachableEntry* out_parsed_block, GetContext* get_context, \ + BlockCacheLookupContext* lookup_context, bool for_compaction, \ + bool use_cache, bool async_read, bool use_block_cache_for_lookup) const; \ + template Status BlockBasedTable::MaybeReadBlockAndLoadToCache( \ + FilePrefetchBuffer * prefetch_buffer, const ReadOptions& ro, \ + const BlockHandle& handle, const UncompressionDict& uncompression_dict, \ + bool for_compaction, CachableEntry* block_entry, \ + GetContext* get_context, BlockCacheLookupContext* lookup_context, \ + BlockContents* contents, bool async_read, \ + bool use_block_cache_for_lookup) const; \ + template Status BlockBasedTable::LookupAndPinBlocksInCache( \ + const ReadOptions& ro, const BlockHandle& handle, \ + CachableEntry* out_parsed_block) const; + +INSTANTIATE_BLOCKLIKE_TEMPLATES(ParsedFullFilterBlock); +INSTANTIATE_BLOCKLIKE_TEMPLATES(UncompressionDict); +INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kData); +INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kIndex); +INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kFilterPartitionIndex); +INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kRangeDeletion); +INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kMetaIndex); } // namespace ROCKSDB_NAMESPACE @@ -676,7 +686,7 @@ Status BlockBasedTable::Open( rep->table_properties->compression_name == CompressionTypeToString(kZSTDNotFinalCompression)); rep->create_context = BlockCreateContext( - &rep->table_options, rep->ioptions.stats, + &rep->table_options, &rep->ioptions, rep->ioptions.stats, blocks_definitely_zstd_compressed, block_protection_bytes_per_key, rep->internal_comparator.user_comparator(), rep->index_value_is_full, rep->index_has_first_key); @@ -771,6 +781,7 @@ Status BlockBasedTable::Open( if (!s.ok()) { return s; } + rep->verify_checksum_set_on_open = ro.verify_checksums; s = new_table->PrefetchIndexAndFilterBlocks( ro, prefetch_buffer.get(), metaindex_iter.get(), new_table.get(), prefetch_all, table_options, level, file_size, @@ -859,10 +870,11 @@ Status BlockBasedTable::PrefetchTail( &prefetch_off_len_pair); #endif // NDEBUG + IOOptions opts; + Status s = file->PrepareIOOptions(ro, opts); // Try file system prefetch - if (!file->use_direct_io() && !force_direct_prefetch) { - if (!file->Prefetch(prefetch_off, prefetch_len, ro.rate_limiter_priority) - .IsNotSupported()) { + if (s.ok() && !file->use_direct_io() && !force_direct_prefetch) { + if (!file->Prefetch(opts, prefetch_off, prefetch_len).IsNotSupported()) { prefetch_buffer->reset(new FilePrefetchBuffer( 0 /* readahead_size */, 0 /* max_readahead_size */, false /* enable */, true /* track_min_offset */)); @@ -876,14 +888,11 @@ Status BlockBasedTable::PrefetchTail( true /* track_min_offset */, false /* implicit_auto_readahead */, 0 /* num_file_reads */, 0 /* num_file_reads_for_auto_readahead */, nullptr /* fs */, nullptr /* clock */, stats, + /* readahead_cb */ nullptr, FilePrefetchBufferUsage::kTableOpenPrefetchTail)); - IOOptions opts; - Status s = file->PrepareIOOptions(ro, opts); if (s.ok()) { - s = (*prefetch_buffer) - ->Prefetch(opts, file, prefetch_off, prefetch_len, - ro.rate_limiter_priority); + s = (*prefetch_buffer)->Prefetch(opts, file, prefetch_off, prefetch_len); } return s; } @@ -990,7 +999,8 @@ Status BlockBasedTable::ReadRangeDelBlock( read_options, range_del_handle, /*input_iter=*/nullptr, BlockType::kRangeDeletion, /*get_context=*/nullptr, lookup_context, prefetch_buffer, - /*for_compaction= */ false, /*async_read= */ false, tmp_status)); + /*for_compaction= */ false, /*async_read= */ false, tmp_status, + /*use_block_cache_for_lookup=*/true)); assert(iter != nullptr); s = iter->status(); if (!s.ok()) { @@ -1214,23 +1224,7 @@ Status BlockBasedTable::PrefetchIndexAndFilterBlocks( return s; } -void BlockBasedTable::SetupForCompaction() { - switch (rep_->ioptions.access_hint_on_compaction_start) { - case Options::NONE: - break; - case Options::NORMAL: - rep_->file->file()->Hint(FSRandomAccessFile::kNormal); - break; - case Options::SEQUENTIAL: - rep_->file->file()->Hint(FSRandomAccessFile::kSequential); - break; - case Options::WILLNEED: - rep_->file->file()->Hint(FSRandomAccessFile::kWillNeed); - break; - default: - assert(false); - } -} +void BlockBasedTable::SetupForCompaction() {} std::shared_ptr BlockBasedTable::GetTableProperties() const { @@ -1313,8 +1307,8 @@ Cache::Priority BlockBasedTable::GetCachePriority() const { template WithBlocklikeCheck BlockBasedTable::GetDataBlockFromCache( const Slice& cache_key, BlockCacheInterface block_cache, - CachableEntry* out_parsed_block, - GetContext* get_context) const { + CachableEntry* out_parsed_block, GetContext* get_context, + const UncompressionDict* dict) const { assert(out_parsed_block); assert(out_parsed_block->IsEmpty()); @@ -1323,10 +1317,12 @@ WithBlocklikeCheck BlockBasedTable::GetDataBlockFromCache( // Lookup uncompressed cache first if (block_cache) { + BlockCreateContext create_ctx = rep_->create_context; + create_ctx.dict = dict; assert(!cache_key.empty()); auto cache_handle = block_cache.LookupFull( - cache_key, &rep_->create_context, GetCachePriority(), - statistics, rep_->ioptions.lowest_used_cache_tier); + cache_key, &create_ctx, GetCachePriority(), statistics, + rep_->ioptions.lowest_used_cache_tier); // Avoid updating metrics here if the handle is not complete yet. This // happens with MultiGet and secondary cache. So update the metrics only @@ -1353,8 +1349,9 @@ WithBlocklikeCheck BlockBasedTable::GetDataBlockFromCache( template WithBlocklikeCheck BlockBasedTable::PutDataBlockToCache( const Slice& cache_key, BlockCacheInterface block_cache, - CachableEntry* out_parsed_block, BlockContents&& block_contents, - CompressionType block_comp_type, + CachableEntry* out_parsed_block, + BlockContents&& uncompressed_block_contents, + BlockContents&& compressed_block_contents, CompressionType block_comp_type, const UncompressionDict& uncompression_dict, MemoryAllocator* memory_allocator, GetContext* get_context) const { const ImmutableOptions& ioptions = rep_->ioptions; @@ -1366,23 +1363,22 @@ WithBlocklikeCheck BlockBasedTable::PutDataBlockToCache( Statistics* statistics = ioptions.stats; std::unique_ptr block_holder; - if (block_comp_type != kNoCompression) { + if (block_comp_type != kNoCompression && + uncompressed_block_contents.data.empty()) { + assert(compressed_block_contents.data.data()); // Retrieve the uncompressed contents into a new buffer - BlockContents uncompressed_block_contents; UncompressionContext context(block_comp_type); UncompressionInfo info(context, uncompression_dict, block_comp_type); - s = UncompressBlockData(info, block_contents.data.data(), - block_contents.data.size(), + s = UncompressBlockData(info, compressed_block_contents.data.data(), + compressed_block_contents.data.size(), &uncompressed_block_contents, format_version, ioptions, memory_allocator); if (!s.ok()) { return s; } - rep_->create_context.Create(&block_holder, - std::move(uncompressed_block_contents)); - } else { - rep_->create_context.Create(&block_holder, std::move(block_contents)); } + rep_->create_context.Create(&block_holder, + std::move(uncompressed_block_contents)); // insert into uncompressed block cache if (block_cache && block_holder->own_bytes()) { @@ -1390,7 +1386,8 @@ WithBlocklikeCheck BlockBasedTable::PutDataBlockToCache( BlockCacheTypedHandle* cache_handle = nullptr; s = block_cache.InsertFull(cache_key, block_holder.get(), charge, &cache_handle, GetCachePriority(), - rep_->ioptions.lowest_used_cache_tier); + rep_->ioptions.lowest_used_cache_tier, + compressed_block_contents.data, block_comp_type); if (s.ok()) { assert(cache_handle != nullptr); @@ -1477,6 +1474,62 @@ IndexBlockIter* BlockBasedTable::InitBlockIterator( block_contents_pinned, rep->user_defined_timestamps_persisted); } +// Right now only called for Data blocks. +template +Status BlockBasedTable::LookupAndPinBlocksInCache( + const ReadOptions& ro, const BlockHandle& handle, + CachableEntry* out_parsed_block) const { + BlockCacheInterface block_cache{ + rep_->table_options.block_cache.get()}; + + assert(block_cache); + + Status s; + CachableEntry uncompression_dict; + if (rep_->uncompression_dict_reader) { + const bool no_io = (ro.read_tier == kBlockCacheTier); + s = rep_->uncompression_dict_reader->GetOrReadUncompressionDictionary( + /* prefetch_buffer= */ nullptr, ro, no_io, ro.verify_checksums, + /* get_context= */ nullptr, /* lookup_context= */ nullptr, + &uncompression_dict); + if (!s.ok()) { + return s; + } + } + + // Do the lookup. + CacheKey key_data = GetCacheKey(rep_->base_cache_key, handle); + const Slice key = key_data.AsSlice(); + + Statistics* statistics = rep_->ioptions.statistics.get(); + + BlockCreateContext create_ctx = rep_->create_context; + create_ctx.dict = uncompression_dict.GetValue() + ? uncompression_dict.GetValue() + : &UncompressionDict::GetEmptyDict(); + + auto cache_handle = + block_cache.LookupFull(key, &create_ctx, GetCachePriority(), + statistics, rep_->ioptions.lowest_used_cache_tier); + + if (!cache_handle) { + UpdateCacheMissMetrics(TBlocklike::kBlockType, /* get_context = */ nullptr); + return s; + } + + // Found in Cache. + TBlocklike* value = block_cache.Value(cache_handle); + if (value) { + UpdateCacheHitMetrics(TBlocklike::kBlockType, /* get_context = */ nullptr, + block_cache.get()->GetUsage(cache_handle)); + } + out_parsed_block->SetCachedValue(value, block_cache.get(), cache_handle); + + assert(!out_parsed_block->IsEmpty()); + + return s; +} + // If contents is nullptr, this function looks up the block caches for the // data block referenced by handle, and read the block from disk if necessary. // If contents is non-null, it skips the cache lookup and disk read, since @@ -1489,12 +1542,12 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache( const BlockHandle& handle, const UncompressionDict& uncompression_dict, bool for_compaction, CachableEntry* out_parsed_block, GetContext* get_context, BlockCacheLookupContext* lookup_context, - BlockContents* contents, bool async_read) const { + BlockContents* contents, bool async_read, + bool use_block_cache_for_lookup) const { assert(out_parsed_block != nullptr); const bool no_io = (ro.read_tier == kBlockCacheTier); BlockCacheInterface block_cache{ rep_->table_options.block_cache.get()}; - // First, try to get the block from the cache // // If either block cache is enabled, we'll try to read from it. @@ -1508,21 +1561,25 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache( key = key_data.AsSlice(); if (!contents) { - s = GetDataBlockFromCache(key, block_cache, out_parsed_block, - get_context); - // Value could still be null at this point, so check the cache handle - // and update the read pattern for prefetching - if (out_parsed_block->GetValue() || out_parsed_block->GetCacheHandle()) { - // TODO(haoyu): Differentiate cache hit on uncompressed block cache and - // compressed block cache. - is_cache_hit = true; - if (prefetch_buffer) { - // Update the block details so that PrefetchBuffer can use the read - // pattern to determine if reads are sequential or not for - // prefetching. It should also take in account blocks read from cache. - prefetch_buffer->UpdateReadPattern( - handle.offset(), BlockSizeWithTrailer(handle), - ro.adaptive_readahead /*decrease_readahead_size*/); + if (use_block_cache_for_lookup) { + s = GetDataBlockFromCache(key, block_cache, out_parsed_block, + get_context, &uncompression_dict); + // Value could still be null at this point, so check the cache handle + // and update the read pattern for prefetching + if (out_parsed_block->GetValue() || + out_parsed_block->GetCacheHandle()) { + // TODO(haoyu): Differentiate cache hit on uncompressed block cache + // and compressed block cache. + is_cache_hit = true; + if (prefetch_buffer) { + // Update the block details so that PrefetchBuffer can use the read + // pattern to determine if reads are sequential or not for + // prefetching. It should also take in account blocks read from + // cache. + prefetch_buffer->UpdateReadPattern( + handle.offset(), BlockSizeWithTrailer(handle), + ro.adaptive_readahead /*decrease_readahead_size*/); + } } } } @@ -1537,14 +1594,26 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache( TBlocklike::kBlockType != BlockType::kFilter && TBlocklike::kBlockType != BlockType::kCompressionDictionary && rep_->blocks_maybe_compressed; + // This flag, if true, tells BlockFetcher to return the uncompressed + // block when ReadBlockContents() is called. const bool do_uncompress = maybe_compressed; CompressionType contents_comp_type; // Maybe serialized or uncompressed BlockContents tmp_contents; + BlockContents uncomp_contents; + BlockContents comp_contents; if (!contents) { Histograms histogram = for_compaction ? READ_BLOCK_COMPACTION_MICROS : READ_BLOCK_GET_MICROS; StopWatch sw(rep_->ioptions.clock, statistics, histogram); + // Setting do_uncompress to false may cause an extra mempcy in the + // following cases - + // 1. Compression is enabled, but block is not actually compressed + // 2. Compressed block is in the prefetch buffer + // 3. Direct IO + // + // It would also cause a memory allocation to be used rather than + // stack if the compressed block size is < 5KB BlockFetcher block_fetcher( rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, &tmp_contents, rep_->ioptions, do_uncompress, maybe_compressed, @@ -1565,7 +1634,6 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache( } contents_comp_type = block_fetcher.get_compression_type(); - contents = &tmp_contents; if (get_context) { switch (TBlocklike::kBlockType) { case BlockType::kIndex: @@ -1579,17 +1647,43 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache( break; } } + if (s.ok()) { + if (do_uncompress && contents_comp_type != kNoCompression) { + comp_contents = BlockContents(block_fetcher.GetCompressedBlock()); + uncomp_contents = std::move(tmp_contents); + } else if (contents_comp_type != kNoCompression) { + // do_uncompress must be false, so output of BlockFetcher is + // compressed + comp_contents = std::move(tmp_contents); + } else { + uncomp_contents = std::move(tmp_contents); + } + + // If filling cache is allowed and a cache is configured, try to put + // the block to the cache. Do this here while block_fetcher is in + // scope, since comp_contents will be a reference to the compressed + // block in block_fetcher + s = PutDataBlockToCache( + key, block_cache, out_parsed_block, std::move(uncomp_contents), + std::move(comp_contents), contents_comp_type, uncompression_dict, + GetMemoryAllocator(rep_->table_options), get_context); + } } else { contents_comp_type = GetBlockCompressionType(*contents); - } + if (contents_comp_type != kNoCompression) { + comp_contents = std::move(*contents); + } else { + uncomp_contents = std::move(*contents); + } - if (s.ok()) { - // If filling cache is allowed and a cache is configured, try to put the - // block to the cache. - s = PutDataBlockToCache( - key, block_cache, out_parsed_block, std::move(*contents), - contents_comp_type, uncompression_dict, - GetMemoryAllocator(rep_->table_options), get_context); + if (s.ok()) { + // If filling cache is allowed and a cache is configured, try to put + // the block to the cache. + s = PutDataBlockToCache( + key, block_cache, out_parsed_block, std::move(uncomp_contents), + std::move(comp_contents), contents_comp_type, uncompression_dict, + GetMemoryAllocator(rep_->table_options), get_context); + } } } } @@ -1705,7 +1799,7 @@ WithBlocklikeCheck BlockBasedTable::RetrieveBlock( const BlockHandle& handle, const UncompressionDict& uncompression_dict, CachableEntry* out_parsed_block, GetContext* get_context, BlockCacheLookupContext* lookup_context, bool for_compaction, - bool use_cache, bool async_read) const { + bool use_cache, bool async_read, bool use_block_cache_for_lookup) const { assert(out_parsed_block); assert(out_parsed_block->IsEmpty()); @@ -1714,7 +1808,7 @@ WithBlocklikeCheck BlockBasedTable::RetrieveBlock( s = MaybeReadBlockAndLoadToCache( prefetch_buffer, ro, handle, uncompression_dict, for_compaction, out_parsed_block, get_context, lookup_context, - /*contents=*/nullptr, async_read); + /*contents=*/nullptr, async_read, use_block_cache_for_lookup); if (!s.ok()) { return s; @@ -1801,7 +1895,8 @@ BlockBasedTable::PartitionedIndexIteratorState::NewSecondaryIterator( rep->internal_comparator.user_comparator(), rep->get_global_seqno(BlockType::kIndex), nullptr, kNullStats, true, rep->index_has_first_key, rep->index_key_includes_seq, - rep->index_value_is_full); + rep->index_value_is_full, /*block_contents_pinned=*/false, + rep->user_defined_timestamps_persisted); } // This will be broken if the user specifies an unusual implementation @@ -1925,6 +2020,16 @@ FragmentedRangeTombstoneIterator* BlockBasedTable::NewRangeTombstoneIterator( snapshot, read_options.timestamp); } +FragmentedRangeTombstoneIterator* BlockBasedTable::NewRangeTombstoneIterator( + SequenceNumber read_seqno, const Slice* timestamp) { + if (rep_->fragmented_range_dels == nullptr) { + return nullptr; + } + return new FragmentedRangeTombstoneIterator(rep_->fragmented_range_dels, + rep_->internal_comparator, + read_seqno, timestamp); +} + bool BlockBasedTable::FullFilterKeyMayMatch( FilterBlockReader* filter, const Slice& internal_key, const bool no_io, const SliceTransform* prefix_extractor, GetContext* get_context, @@ -2167,7 +2272,8 @@ Status BlockBasedTable::Get(const ReadOptions& read_options, const Slice& key, NewDataBlockIterator( read_options, v.handle, &biter, BlockType::kData, get_context, &lookup_data_block_context, /*prefetch_buffer=*/nullptr, - /*for_compaction=*/false, /*async_read=*/false, tmp_status); + /*for_compaction=*/false, /*async_read=*/false, tmp_status, + /*use_block_cache_for_lookup=*/true); if (no_io && biter.status().IsIncomplete()) { // couldn't get block from block_cache @@ -2337,7 +2443,7 @@ Status BlockBasedTable::Prefetch(const ReadOptions& read_options, read_options, block_handle, &biter, /*type=*/BlockType::kData, /*get_context=*/nullptr, &lookup_context, /*prefetch_buffer=*/nullptr, /*for_compaction=*/false, - /*async_read=*/false, tmp_status); + /*async_read=*/false, tmp_status, /*use_block_cache_for_lookup=*/true); if (!biter.status().ok()) { // there was an unexpected error while pre-fetching @@ -2454,6 +2560,10 @@ BlockType BlockBasedTable::GetBlockTypeForMetaBlockByName( return BlockType::kHashIndexMetadata; } + if (meta_block_name == kIndexBlockName) { + return BlockType::kIndex; + } + if (meta_block_name.starts_with(kObsoleteFilterBlockPrefix)) { // Obsolete but possible in old files return BlockType::kInvalid; @@ -2474,6 +2584,9 @@ Status BlockBasedTable::VerifyChecksumInMetaBlocks( BlockHandle handle; Slice input = index_iter->value(); s = handle.DecodeFrom(&input); + if (!s.ok()) { + break; + } BlockContents contents; const Slice meta_block_name = index_iter->key(); if (meta_block_name == kPropertiesBlockName) { @@ -2484,7 +2597,13 @@ Status BlockBasedTable::VerifyChecksumInMetaBlocks( nullptr /* prefetch_buffer */, rep_->footer, rep_->ioptions, &table_properties, nullptr /* memory_allocator */); + } else if (rep_->verify_checksum_set_on_open && + meta_block_name == kIndexBlockName) { + // WART: For now, to maintain similar I/O behavior as before + // format_version=6, we skip verifying index block checksum--but only + // if it was checked on open. } else { + // FIXME? Need to verify checksums of index and filter partitions? s = BlockFetcher( rep_->file.get(), nullptr /* prefetch buffer */, rep_->footer, read_options, handle, &contents, rep_->ioptions, @@ -2526,11 +2645,23 @@ bool BlockBasedTable::TEST_KeyInCache(const ReadOptions& options, options, /*need_upper_bound_check=*/false, /*input_iter=*/nullptr, /*get_context=*/nullptr, /*lookup_context=*/nullptr)); iiter->Seek(key); + assert(iiter->status().ok()); assert(iiter->Valid()); return TEST_BlockInCache(iiter->value().handle); } +void BlockBasedTable::TEST_GetDataBlockHandle(const ReadOptions& options, + const Slice& key, + BlockHandle& handle) { + std::unique_ptr> iiter(NewIndexIterator( + options, /*disable_prefix_seek=*/false, /*input_iter=*/nullptr, + /*get_context=*/nullptr, /*lookup_context=*/nullptr)); + iiter->Seek(key); + assert(iiter->Valid()); + handle = iiter->value().handle; +} + // REQUIRES: The following fields of rep_ should have already been populated: // 1. file // 2. index_handle, @@ -2542,6 +2673,15 @@ Status BlockBasedTable::CreateIndexReader( InternalIterator* meta_iter, bool use_cache, bool prefetch, bool pin, BlockCacheLookupContext* lookup_context, std::unique_ptr* index_reader) { + if (FormatVersionUsesIndexHandleInFooter(rep_->footer.format_version())) { + rep_->index_handle = rep_->footer.index_handle(); + } else { + Status s = FindMetaBlock(meta_iter, kIndexBlockName, &rep_->index_handle); + if (!s.ok()) { + return s; + } + } + switch (rep_->index_type) { case BlockBasedTableOptions::kTwoLevelIndexSearch: { return PartitionIndexReader::Create(this, ro, prefetch_buffer, use_cache, @@ -2709,7 +2849,7 @@ bool BlockBasedTable::TEST_FilterBlockInCache() const { bool BlockBasedTable::TEST_IndexBlockInCache() const { assert(rep_ != nullptr); - return TEST_BlockInCache(rep_->footer.index_handle()); + return TEST_BlockInCache(rep_->index_handle); } Status BlockBasedTable::GetKVPairsFromDataBlocks( @@ -2740,7 +2880,7 @@ Status BlockBasedTable::GetKVPairsFromDataBlocks( /*input_iter=*/nullptr, /*type=*/BlockType::kData, /*get_context=*/nullptr, /*lookup_context=*/nullptr, /*prefetch_buffer=*/nullptr, /*for_compaction=*/false, - /*async_read=*/false, tmp_status)); + /*async_read=*/false, tmp_status, /*use_block_cache_for_lookup=*/true)); s = datablock_iter->status(); if (!s.ok()) { @@ -2924,7 +3064,7 @@ Status BlockBasedTable::DumpIndexBlock(std::ostream& out_stream) { << " size " << blockhandles_iter->value().handle.size() << "\n"; std::string str_key = user_key.ToString(); - std::string res_key(""); + std::string res_key; char cspace = ' '; for (size_t i = 0; i < str_key.size(); i++) { res_key.append(&str_key[i], 1); @@ -2979,7 +3119,7 @@ Status BlockBasedTable::DumpDataBlocks(std::ostream& out_stream) { /*input_iter=*/nullptr, /*type=*/BlockType::kData, /*get_context=*/nullptr, /*lookup_context=*/nullptr, /*prefetch_buffer=*/nullptr, /*for_compaction=*/false, - /*async_read=*/false, tmp_status)); + /*async_read=*/false, tmp_status, /*use_block_cache_for_lookup=*/true)); s = datablock_iter->status(); if (!s.ok()) { @@ -3025,7 +3165,7 @@ void BlockBasedTable::DumpKeyValue(const Slice& key, const Slice& value, std::string str_key = ikey.user_key().ToString(); std::string str_value = value.ToString(); - std::string res_key(""), res_value(""); + std::string res_key, res_value; char cspace = ' '; for (size_t i = 0; i < str_key.size(); i++) { if (str_key[i] == '\0') { diff --git a/table/block_based/block_based_table_reader.h b/table/block_based/block_based_table_reader.h index d0686a4bc..a7a94cd0b 100644 --- a/table/block_based/block_based_table_reader.h +++ b/table/block_based/block_based_table_reader.h @@ -138,6 +138,9 @@ class BlockBasedTable : public TableReader { FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator( const ReadOptions& read_options) override; + FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator( + SequenceNumber read_seqno, const Slice* timestamp) override; + // @param skip_filters Disables loading/accessing the filter block Status Get(const ReadOptions& readOptions, const Slice& key, GetContext* get_context, const SliceTransform* prefix_extractor, @@ -185,6 +188,9 @@ class BlockBasedTable : public TableReader { // REQUIRES: key is in this table && block cache enabled bool TEST_KeyInCache(const ReadOptions& options, const Slice& key); + void TEST_GetDataBlockHandle(const ReadOptions& options, const Slice& key, + BlockHandle& handle); + // Set up the table for Compaction. Might change some parameters with // posix_fadvise void SetupForCompaction() override; @@ -277,6 +283,11 @@ class BlockBasedTable : public TableReader { Status GetKVPairsFromDataBlocks(const ReadOptions& read_options, std::vector* kv_pair_blocks); + template + Status LookupAndPinBlocksInCache( + const ReadOptions& ro, const BlockHandle& handle, + CachableEntry* out_parsed_block) const; + struct Rep; Rep* get_rep() { return rep_; } @@ -284,14 +295,12 @@ class BlockBasedTable : public TableReader { // input_iter: if it is not null, update this one and return it as Iterator template - TBlockIter* NewDataBlockIterator(const ReadOptions& ro, - const BlockHandle& block_handle, - TBlockIter* input_iter, BlockType block_type, - GetContext* get_context, - BlockCacheLookupContext* lookup_context, - FilePrefetchBuffer* prefetch_buffer, - bool for_compaction, bool async_read, - Status& s) const; + TBlockIter* NewDataBlockIterator( + const ReadOptions& ro, const BlockHandle& block_handle, + TBlockIter* input_iter, BlockType block_type, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + FilePrefetchBuffer* prefetch_buffer, bool for_compaction, bool async_read, + Status& s, bool use_block_cache_for_lookup) const; // input_iter: if it is not null, update this one and return it as Iterator template @@ -348,7 +357,8 @@ class BlockBasedTable : public TableReader { const BlockHandle& handle, const UncompressionDict& uncompression_dict, bool for_compaction, CachableEntry* block_entry, GetContext* get_context, BlockCacheLookupContext* lookup_context, - BlockContents* contents, bool async_read) const; + BlockContents* contents, bool async_read, + bool use_block_cache_for_lookup) const; // Similar to the above, with one crucial difference: it will retrieve the // block from the file even if there are no caches configured (assuming the @@ -359,7 +369,7 @@ class BlockBasedTable : public TableReader { const BlockHandle& handle, const UncompressionDict& uncompression_dict, CachableEntry* block_entry, GetContext* get_context, BlockCacheLookupContext* lookup_context, bool for_compaction, - bool use_cache, bool async_read) const; + bool use_cache, bool async_read, bool use_block_cache_for_lookup) const; template WithBlocklikeCheck SaveLookupContextOrTraceRecord( @@ -408,7 +418,8 @@ class BlockBasedTable : public TableReader { template WithBlocklikeCheck GetDataBlockFromCache( const Slice& cache_key, BlockCacheInterface block_cache, - CachableEntry* block, GetContext* get_context) const; + CachableEntry* block, GetContext* get_context, + const UncompressionDict* dict) const; // Put a maybe compressed block to the corresponding block caches. // This method will perform decompression against block_contents if needed @@ -423,7 +434,9 @@ class BlockBasedTable : public TableReader { template WithBlocklikeCheck PutDataBlockToCache( const Slice& cache_key, BlockCacheInterface block_cache, - CachableEntry* cached_block, BlockContents&& block_contents, + CachableEntry* cached_block, + BlockContents&& uncompressed_block_contents, + BlockContents&& compressed_block_contents, CompressionType block_comp_type, const UncompressionDict& uncompression_dict, MemoryAllocator* memory_allocator, GetContext* get_context) const; @@ -594,6 +607,7 @@ struct BlockBasedTable::Rep { BlockHandle compression_dict_handle; std::shared_ptr table_properties; + BlockHandle index_handle; BlockBasedTableOptions::IndexType index_type; bool whole_key_filtering; bool prefix_filtering; @@ -637,6 +651,12 @@ struct BlockBasedTable::Rep { bool index_key_includes_seq = true; bool index_value_is_full = true; + // Whether block checksums in metadata blocks were verified on open. + // This is only to mostly maintain current dubious behavior of VerifyChecksum + // with respect to index blocks, but only when the checksum was previously + // verified. + bool verify_checksum_set_on_open = false; + const bool immortal_table; // Whether the user key contains user-defined timestamps. If this is false and // the running user comparator has a non-zero timestamp size, a min timestamp @@ -678,25 +698,30 @@ struct BlockBasedTable::Rep { void CreateFilePrefetchBuffer( size_t readahead_size, size_t max_readahead_size, std::unique_ptr* fpb, bool implicit_auto_readahead, - uint64_t num_file_reads, - uint64_t num_file_reads_for_auto_readahead) const { + uint64_t num_file_reads, uint64_t num_file_reads_for_auto_readahead, + + const std::function& readaheadsize_cb, + FilePrefetchBufferUsage usage) const { fpb->reset(new FilePrefetchBuffer( readahead_size, max_readahead_size, !ioptions.allow_mmap_reads /* enable */, false /* track_min_offset */, implicit_auto_readahead, num_file_reads, num_file_reads_for_auto_readahead, ioptions.fs.get(), ioptions.clock, - ioptions.stats)); + ioptions.stats, readaheadsize_cb, usage)); } void CreateFilePrefetchBufferIfNotExists( size_t readahead_size, size_t max_readahead_size, std::unique_ptr* fpb, bool implicit_auto_readahead, - uint64_t num_file_reads, - uint64_t num_file_reads_for_auto_readahead) const { + uint64_t num_file_reads, uint64_t num_file_reads_for_auto_readahead, + + const std::function& readaheadsize_cb, + FilePrefetchBufferUsage usage = FilePrefetchBufferUsage::kUnknown) const { if (!(*fpb)) { CreateFilePrefetchBuffer(readahead_size, max_readahead_size, fpb, implicit_auto_readahead, num_file_reads, - num_file_reads_for_auto_readahead); + num_file_reads_for_auto_readahead, + readaheadsize_cb, usage); } } diff --git a/table/block_based/block_based_table_reader_impl.h b/table/block_based/block_based_table_reader_impl.h index 801b4614f..fedccd5ee 100644 --- a/table/block_based/block_based_table_reader_impl.h +++ b/table/block_based/block_based_table_reader_impl.h @@ -49,7 +49,7 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator( BlockType block_type, GetContext* get_context, BlockCacheLookupContext* lookup_context, FilePrefetchBuffer* prefetch_buffer, bool for_compaction, bool async_read, - Status& s) const { + Status& s, bool use_block_cache_for_lookup) const { using IterBlocklike = typename IterTraits::IterBlocklike; PERF_TIMER_GUARD(new_table_block_iter_nanos); @@ -67,9 +67,13 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator( // might already be under way and this would invalidate it. Also, the // uncompression dict is typically at the end of the file and would // most likely break the sequentiality of the access pattern. + // Same is with auto_readahead_size. It iterates over index to lookup for + // data blocks. And this could break the the sequentiality of the access + // pattern. s = rep_->uncompression_dict_reader->GetOrReadUncompressionDictionary( - ro.async_io ? nullptr : prefetch_buffer, ro, no_io, ro.verify_checksums, - get_context, lookup_context, &uncompression_dict); + ((ro.async_io || ro.auto_readahead_size) ? nullptr : prefetch_buffer), + ro, no_io, ro.verify_checksums, get_context, lookup_context, + &uncompression_dict); if (!s.ok()) { iter->Invalidate(s); return iter; @@ -77,15 +81,15 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator( const UncompressionDict& dict = uncompression_dict.GetValue() ? *uncompression_dict.GetValue() : UncompressionDict::GetEmptyDict(); - s = RetrieveBlock(prefetch_buffer, ro, handle, dict, - &block.As(), get_context, lookup_context, - for_compaction, - /* use_cache */ true, async_read); + s = RetrieveBlock( + prefetch_buffer, ro, handle, dict, &block.As(), + get_context, lookup_context, for_compaction, + /* use_cache */ true, async_read, use_block_cache_for_lookup); } else { s = RetrieveBlock( prefetch_buffer, ro, handle, UncompressionDict::GetEmptyDict(), &block.As(), get_context, lookup_context, for_compaction, - /* use_cache */ true, async_read); + /* use_cache */ true, async_read, use_block_cache_for_lookup); } if (s.IsTryAgain() && async_read) { diff --git a/table/block_based/block_based_table_reader_sync_and_async.h b/table/block_based/block_based_table_reader_sync_and_async.h index eb4bf5bed..e7621909c 100644 --- a/table/block_based/block_based_table_reader_sync_and_async.h +++ b/table/block_based/block_based_table_reader_sync_and_async.h @@ -50,12 +50,12 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) } // XXX: use_cache=true means double cache query? - statuses[idx_in_batch] = - RetrieveBlock(nullptr, options, handle, uncompression_dict, - &results[idx_in_batch].As(), - mget_iter->get_context, /* lookup_context */ nullptr, - /* for_compaction */ false, /* use_cache */ true, - /* async_read */ false); + statuses[idx_in_batch] = RetrieveBlock( + nullptr, options, handle, uncompression_dict, + &results[idx_in_batch].As(), mget_iter->get_context, + /* lookup_context */ nullptr, + /* for_compaction */ false, /* use_cache */ true, + /* async_read */ false, /* use_block_cache_for_lookup */ true); } assert(idx_in_batch == handles->size()); CO_RETURN; @@ -144,7 +144,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) if (file->use_direct_io()) { #endif // WITH_COROUTINES s = file->MultiRead(opts, &read_reqs[0], read_reqs.size(), - &direct_io_buf, options.rate_limiter_priority); + &direct_io_buf); #if defined(WITH_COROUTINES) } else { co_await batch->context()->reader().MultiReadAsync( @@ -222,9 +222,8 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) // begin address of each read request, we need to add the offset // in each read request. Checksum is stored in the block trailer, // beyond the payload size. - s = VerifyBlockChecksum(footer.checksum_type(), data + req_offset, - handle.size(), rep_->file->file_name(), - handle.offset()); + s = VerifyBlockChecksum(footer, data + req_offset, handle.size(), + rep_->file->file_name(), handle.offset()); TEST_SYNC_POINT_CALLBACK("RetrieveMultipleBlocks:VerifyChecksum", &s); } } else if (!use_shared_buffer) { @@ -270,7 +269,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) nullptr, options, handle, uncompression_dict, /*for_compaction=*/false, block_entry, mget_iter->get_context, /*lookup_context=*/nullptr, &serialized_block, - /*async_read=*/false); + /*async_read=*/false, /*use_block_cache_for_lookup=*/true); // block_entry value could be null if no block cache is present, i.e // BlockBasedTableOptions::no_block_cache is true and no compressed @@ -403,6 +402,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet) BCI block_cache{rep_->table_options.block_cache.get()}; std::array async_handles; + BlockCreateContext create_ctx = rep_->create_context; std::array cache_keys; size_t cache_lookup_count = 0; @@ -449,6 +449,9 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet) sst_file_range.SkipKey(miter); continue; } + create_ctx.dict = uncompression_dict.GetValue() + ? uncompression_dict.GetValue() + : &UncompressionDict::GetEmptyDict(); if (v.handle.offset() == prev_offset) { // This key can reuse the previous block (later on). @@ -476,7 +479,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet) GetCacheKey(rep_->base_cache_key, v.handle); async_handle.key = cache_keys[cache_lookup_count].AsSlice(); // NB: StartAsyncLookupFull populates async_handle.helper - async_handle.create_context = &rep_->create_context; + async_handle.create_context = &create_ctx; async_handle.priority = GetCachePriority(); async_handle.stats = rep_->ioptions.statistics.get(); @@ -629,7 +632,8 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet) read_options, iiter->value().handle, &next_biter, BlockType::kData, get_context, lookup_data_block_context, /* prefetch_buffer= */ nullptr, /* for_compaction = */ false, - /*async_read = */ false, tmp_s); + /*async_read = */ false, tmp_s, + /* use_block_cache_for_lookup = */ true); biter = &next_biter; reusing_prev_block = false; later_reused = false; diff --git a/table/block_based/block_based_table_reader_test.cc b/table/block_based/block_based_table_reader_test.cc index 2aaf505f8..254546893 100644 --- a/table/block_based/block_based_table_reader_test.cc +++ b/table/block_based/block_based_table_reader_test.cc @@ -30,22 +30,42 @@ namespace ROCKSDB_NAMESPACE { class BlockBasedTableReaderBaseTest : public testing::Test { + public: + static constexpr int kBytesPerEntry = 256; + // 16 = (default block size) 4 * 1024 / kBytesPerEntry + static constexpr int kEntriesPerBlock = 16; + protected: // Prepare key-value pairs to occupy multiple blocks. - // Each value is 256B, every 16 pairs constitute 1 block. + // Each (key, value) pair is `kBytesPerEntry` byte, every kEntriesPerBlock + // pairs constitute 1 block. // If mixed_with_human_readable_string_value == true, // then adjacent blocks contain values with different compression // complexity: human readable strings are easier to compress than random - // strings. - static std::map GenerateKVMap( - int num_block = 100, bool mixed_with_human_readable_string_value = false, - size_t ts_sz = 0) { - std::map kv; - + // strings. key is an internal key. + // When ts_sz > 0 and `same_key_diff_ts` is true, this + // function generate keys with the same user provided key, with different + // user defined timestamps and different sequence number to differentiate them + static std::vector> GenerateKVMap( + int num_block = 2, bool mixed_with_human_readable_string_value = false, + size_t ts_sz = 0, bool same_key_diff_ts = false) { + std::vector> kv; + + SequenceNumber seq_no = 0; + uint64_t current_udt = 0; + if (same_key_diff_ts) { + // These numbers are based on the number of keys to create + an arbitrary + // buffer number (100) to avoid overflow. + current_udt = kEntriesPerBlock * num_block + 100; + seq_no = kEntriesPerBlock * num_block + 100; + } Random rnd(101); uint32_t key = 0; + // To make each (key, value) pair occupy exactly kBytesPerEntry bytes. + int value_size = kBytesPerEntry - (8 + static_cast(ts_sz) + + static_cast(kNumInternalBytes)); for (int block = 0; block < num_block; block++) { - for (int i = 0; i < 16; i++) { + for (int i = 0; i < kEntriesPerBlock; i++) { char k[9] = {0}; // Internal key is constructed directly from this key, // and internal key size is required to be >= 8 bytes, @@ -53,19 +73,27 @@ class BlockBasedTableReaderBaseTest : public testing::Test { snprintf(k, sizeof(k), "%08u", key); std::string v; if (mixed_with_human_readable_string_value) { - v = (block % 2) ? rnd.HumanReadableString(256) - : rnd.RandomString(256); + v = (block % 2) ? rnd.HumanReadableString(value_size) + : rnd.RandomString(value_size); } else { - v = rnd.RandomString(256); + v = rnd.RandomString(value_size); } + std::string user_key = std::string(k); if (ts_sz > 0) { - std::string user_key; - AppendKeyWithMinTimestamp(&user_key, std::string(k), ts_sz); - kv[user_key] = v; + if (same_key_diff_ts) { + PutFixed64(&user_key, current_udt); + current_udt -= 1; + } else { + PutFixed64(&user_key, 0); + } + } + InternalKey internal_key(user_key, seq_no, ValueType::kTypeValue); + kv.emplace_back(internal_key.Encode().ToString(), v); + if (same_key_diff_ts) { + seq_no -= 1; } else { - kv[std::string(k)] = v; + key++; } - key++; } } return kv; @@ -88,7 +116,7 @@ class BlockBasedTableReaderBaseTest : public testing::Test { void CreateTable(const std::string& table_name, const ImmutableOptions& ioptions, const CompressionType& compression_type, - const std::map& kv, + const std::vector>& kv, uint32_t compression_parallel_threads = 1, uint32_t compression_dict_bytes = 0) { std::unique_ptr writer; @@ -115,9 +143,8 @@ class BlockBasedTableReaderBaseTest : public testing::Test { // Build table. for (auto it = kv.begin(); it != kv.end(); it++) { - std::string k = ToInternalKey(it->first); std::string v = it->second; - table_builder->Add(k, v); + table_builder->Add(it->first, v); } ASSERT_OK(table_builder->Finish()); } @@ -169,11 +196,6 @@ class BlockBasedTableReaderBaseTest : public testing::Test { std::shared_ptr fs_; Options options_; - std::string ToInternalKey(const std::string& key) { - InternalKey internal_key(key, 0, ValueType::kTypeValue); - return internal_key.Encode().ToString(); - } - private: void WriteToFile(const std::string& content, const std::string& filename) { std::unique_ptr f; @@ -211,11 +233,18 @@ class BlockBasedTableReaderBaseTest : public testing::Test { // Param 7: CompressionOptions.max_dict_bytes and // CompressionOptions.max_dict_buffer_bytes to enable/disable // compression dictionary. +// Param 8: test mode to specify the pattern for generating key / value. When +// true, generate keys with the same user provided key, different +// user-defined timestamps (if udt enabled), different sequence +// numbers. This test mode is used for testing `Get`. When false, +// generate keys with different user provided key, same user-defined +// timestamps (if udt enabled), same sequence number. This test mode is +// used for testing `Get`, `MultiGet`, and `NewIterator`. class BlockBasedTableReaderTest : public BlockBasedTableReaderBaseTest, public testing::WithParamInterface> { + test::UserDefinedTimestampTestMode, uint32_t, uint32_t, bool>> { protected: void SetUp() override { compression_type_ = std::get<0>(GetParam()); @@ -225,6 +254,7 @@ class BlockBasedTableReaderTest persist_udt_ = test::ShouldPersistUDT(udt_test_mode); compression_parallel_threads_ = std::get<5>(GetParam()); compression_dict_bytes_ = std::get<6>(GetParam()); + same_key_diff_ts_ = std::get<7>(GetParam()); BlockBasedTableReaderBaseTest::SetUp(); } @@ -236,6 +266,7 @@ class BlockBasedTableReaderTest opts.partition_filters = opts.index_type == BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + opts.metadata_cache_options.partition_pinning = PinningTier::kAll; options_.table_factory.reset( static_cast(NewBlockBasedTableFactory(opts))); options_.prefix_extractor = @@ -248,8 +279,72 @@ class BlockBasedTableReaderTest bool persist_udt_; uint32_t compression_parallel_threads_; uint32_t compression_dict_bytes_; + bool same_key_diff_ts_; }; +class BlockBasedTableReaderGetTest : public BlockBasedTableReaderTest {}; + +TEST_P(BlockBasedTableReaderGetTest, Get) { + Options options; + if (udt_enabled_) { + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + } + options.persist_user_defined_timestamps = persist_udt_; + size_t ts_sz = options.comparator->timestamp_size(); + std::vector> kv = + BlockBasedTableReaderBaseTest::GenerateKVMap( + 100 /* num_block */, + true /* mixed_with_human_readable_string_value */, ts_sz, + same_key_diff_ts_); + + std::string table_name = "BlockBasedTableReaderGetTest_Get" + + CompressionTypeToString(compression_type_); + + ImmutableOptions ioptions(options); + CreateTable(table_name, ioptions, compression_type_, kv, + compression_parallel_threads_, compression_dict_bytes_); + + std::unique_ptr table; + FileOptions foptions; + foptions.use_direct_reads = use_direct_reads_; + InternalKeyComparator comparator(options.comparator); + NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table, + true /* prefetch_index_and_filter_in_cache */, + nullptr /* status */, persist_udt_); + + ReadOptions read_opts; + ASSERT_OK( + table->VerifyChecksum(read_opts, TableReaderCaller::kUserVerifyChecksum)); + + for (size_t i = 0; i < kv.size(); i += 1) { + Slice key = kv[i].first; + Slice lkey = key; + std::string lookup_ikey; + if (udt_enabled_ && !persist_udt_) { + // When user-defined timestamps are collapsed to be the minimum timestamp, + // we also read with the minimum timestamp to be able to retrieve each + // value. + ReplaceInternalKeyWithMinTimestamp(&lookup_ikey, key, ts_sz); + lkey = lookup_ikey; + } + // Reading the first entry in a block caches the whole block. + if (i % kEntriesPerBlock == 0) { + ASSERT_FALSE(table->TEST_KeyInCache(read_opts, lkey.ToString())); + } else { + ASSERT_TRUE(table->TEST_KeyInCache(read_opts, lkey.ToString())); + } + PinnableSlice value; + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, ExtractUserKey(key), &value, + nullptr, nullptr, nullptr, nullptr, + true /* do_merge */, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr); + ASSERT_OK(table->Get(read_opts, lkey, &get_context, nullptr)); + ASSERT_EQ(value.ToString(), kv[i].second); + ASSERT_TRUE(table->TEST_KeyInCache(read_opts, lkey.ToString())); + } +} + // Tests MultiGet in both direct IO and non-direct IO mode. // The keys should be in cache after MultiGet. TEST_P(BlockBasedTableReaderTest, MultiGet) { @@ -263,7 +358,7 @@ TEST_P(BlockBasedTableReaderTest, MultiGet) { } options.persist_user_defined_timestamps = persist_udt_; size_t ts_sz = options.comparator->timestamp_size(); - std::map kv = + std::vector> kv = BlockBasedTableReaderBaseTest::GenerateKVMap( 100 /* num_block */, true /* mixed_with_human_readable_string_value */, ts_sz); @@ -273,6 +368,8 @@ TEST_P(BlockBasedTableReaderTest, MultiGet) { autovector keys_without_timestamps; autovector values; autovector statuses; + autovector + expected_values; { const int step = static_cast(kv.size()) / MultiGetContext::MAX_BATCH_SIZE; @@ -280,13 +377,15 @@ TEST_P(BlockBasedTableReaderTest, MultiGet) { for (int i = 0; i < MultiGetContext::MAX_BATCH_SIZE; i++) { keys.emplace_back(it->first); if (ts_sz > 0) { - Slice ukey_without_ts = StripTimestampFromUserKey(it->first, ts_sz); + Slice ukey_without_ts = + ExtractUserKeyAndStripTimestamp(it->first, ts_sz); keys_without_timestamps.push_back(ukey_without_ts); } else { - keys_without_timestamps.emplace_back(it->first); + keys_without_timestamps.emplace_back(ExtractUserKey(it->first)); } values.emplace_back(); statuses.emplace_back(); + expected_values.push_back(&(it->second)); std::advance(it, step); } } @@ -311,8 +410,7 @@ TEST_P(BlockBasedTableReaderTest, MultiGet) { // Ensure that keys are not in cache before MultiGet. for (auto& key : keys) { - std::string ikey = ToInternalKey(key.ToString()); - ASSERT_FALSE(table->TEST_KeyInCache(read_opts, ikey)); + ASSERT_FALSE(table->TEST_KeyInCache(read_opts, key.ToString())); } // Prepare MultiGetContext. @@ -321,8 +419,8 @@ TEST_P(BlockBasedTableReaderTest, MultiGet) { autovector sorted_keys; for (size_t i = 0; i < keys.size(); ++i) { get_context.emplace_back(options.comparator, nullptr, nullptr, nullptr, - GetContext::kNotFound, keys[i], &values[i], - nullptr, nullptr, nullptr, nullptr, + GetContext::kNotFound, ExtractUserKey(keys[i]), + &values[i], nullptr, nullptr, nullptr, nullptr, true /* do_merge */, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr); key_context.emplace_back(nullptr, keys_without_timestamps[i], &values[i], @@ -352,9 +450,8 @@ TEST_P(BlockBasedTableReaderTest, MultiGet) { } // Check that keys are in cache after MultiGet. for (size_t i = 0; i < keys.size(); i++) { - std::string ikey = ToInternalKey(keys[i].ToString()); - ASSERT_TRUE(table->TEST_KeyInCache(read_opts, ikey)); - ASSERT_EQ(values[i].ToString(), kv[keys[i].ToString()]); + ASSERT_TRUE(table->TEST_KeyInCache(read_opts, keys[i])); + ASSERT_EQ(values[i].ToString(), *expected_values[i]); } } @@ -369,7 +466,7 @@ TEST_P(BlockBasedTableReaderTest, NewIterator) { } options.persist_user_defined_timestamps = persist_udt_; size_t ts_sz = options.comparator->timestamp_size(); - std::map kv = + std::vector> kv = BlockBasedTableReaderBaseTest::GenerateKVMap( 100 /* num_block */, true /* mixed_with_human_readable_string_value */, ts_sz); @@ -401,8 +498,7 @@ TEST_P(BlockBasedTableReaderTest, NewIterator) { iter->SeekToFirst(); ASSERT_OK(iter->status()); for (auto kv_iter = kv.begin(); kv_iter != kv.end(); kv_iter++) { - std::string ikey = ToInternalKey(kv_iter->first); - ASSERT_EQ(iter->key().ToString(), ikey); + ASSERT_EQ(iter->key().ToString(), kv_iter->first); ASSERT_EQ(iter->value().ToString(), kv_iter->second); iter->Next(); ASSERT_OK(iter->status()); @@ -414,8 +510,7 @@ TEST_P(BlockBasedTableReaderTest, NewIterator) { iter->SeekToLast(); ASSERT_OK(iter->status()); for (auto kv_iter = kv.rbegin(); kv_iter != kv.rend(); kv_iter++) { - std::string ikey = ToInternalKey(kv_iter->first); - ASSERT_EQ(iter->key().ToString(), ikey); + ASSERT_EQ(iter->key().ToString(), kv_iter->first); ASSERT_EQ(iter->value().ToString(), kv_iter->second); iter->Prev(); ASSERT_OK(iter->status()); @@ -504,7 +599,7 @@ class ChargeTableReaderTest TargetCacheChargeTrackingCache> table_reader_charge_tracking_cache_; std::size_t approx_table_reader_mem_; - std::map kv_; + std::vector> kv_; CompressionType compression_type_; private: @@ -641,7 +736,7 @@ TEST_P(BlockBasedTableReaderTestVerifyChecksum, ChecksumMismatch) { } options.persist_user_defined_timestamps = persist_udt_; size_t ts_sz = options.comparator->timestamp_size(); - std::map kv = + std::vector> kv = BlockBasedTableReaderBaseTest::GenerateKVMap( 800 /* num_block */, false /* mixed_with_human_readable_string_value=*/, ts_sz); @@ -705,6 +800,7 @@ TEST_P(BlockBasedTableReaderTestVerifyChecksum, ChecksumMismatch) { // Param 7: CompressionOptions.max_dict_bytes and // CompressionOptions.max_dict_buffer_bytes. This enable/disables // compression dictionary. +// Param 8: test mode to specify the pattern for generating key / value pairs. INSTANTIATE_TEST_CASE_P( BlockBasedTableReaderTest, BlockBasedTableReaderTest, ::testing::Combine( @@ -715,7 +811,20 @@ INSTANTIATE_TEST_CASE_P( BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch, BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey), ::testing::Values(false), ::testing::ValuesIn(test::GetUDTTestModes()), - ::testing::Values(1, 2), ::testing::Values(0, 4096))); + ::testing::Values(1, 2), ::testing::Values(0, 4096), + ::testing::Values(false))); +INSTANTIATE_TEST_CASE_P( + BlockBasedTableReaderGetTest, BlockBasedTableReaderGetTest, + ::testing::Combine( + ::testing::ValuesIn(GetSupportedCompressions()), ::testing::Bool(), + ::testing::Values( + BlockBasedTableOptions::IndexType::kBinarySearch, + BlockBasedTableOptions::IndexType::kHashSearch, + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch, + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey), + ::testing::Values(false), ::testing::ValuesIn(test::GetUDTTestModes()), + ::testing::Values(1, 2), ::testing::Values(0, 4096), + ::testing::Values(false, true))); INSTANTIATE_TEST_CASE_P( VerifyChecksum, BlockBasedTableReaderTestVerifyChecksum, ::testing::Combine( @@ -724,7 +833,8 @@ INSTANTIATE_TEST_CASE_P( ::testing::Values( BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch), ::testing::Values(true), ::testing::ValuesIn(test::GetUDTTestModes()), - ::testing::Values(1, 2), ::testing::Values(0))); + ::testing::Values(1, 2), ::testing::Values(0), + ::testing::Values(false))); } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_builder.cc b/table/block_based/block_builder.cc index 877df81c1..e4950e435 100644 --- a/table/block_based/block_builder.cc +++ b/table/block_based/block_builder.cc @@ -33,9 +33,8 @@ #include "table/block_based/block_builder.h" -#include - #include +#include #include "db/dbformat.h" #include "rocksdb/comparator.h" diff --git a/table/block_based/block_cache.cc b/table/block_based/block_cache.cc index a252899d2..08f5d2158 100644 --- a/table/block_based/block_cache.cc +++ b/table/block_based/block_cache.cc @@ -5,6 +5,8 @@ #include "table/block_based/block_cache.h" +#include "table/block_based/block_based_table_reader.h" + namespace ROCKSDB_NAMESPACE { void BlockCreateContext::Create(std::unique_ptr* parsed_out, @@ -96,7 +98,7 @@ const std::array CacheTier::kVolatileTier) { return kCacheItemFullHelperForBlockType[static_cast(block_type)]; } else { return kCacheItemBasicHelperForBlockType[static_cast(block_type)]; diff --git a/table/block_based/block_cache.h b/table/block_based/block_cache.h index 00eaface3..06ba50566 100644 --- a/table/block_based/block_cache.h +++ b/table/block_based/block_cache.h @@ -70,24 +70,28 @@ class Block_kMetaIndex : public Block { struct BlockCreateContext : public Cache::CreateContext { BlockCreateContext() {} BlockCreateContext(const BlockBasedTableOptions* _table_options, - Statistics* _statistics, bool _using_zstd, - uint8_t _protection_bytes_per_key, + const ImmutableOptions* _ioptions, Statistics* _statistics, + bool _using_zstd, uint8_t _protection_bytes_per_key, const Comparator* _raw_ucmp, bool _index_value_is_full = false, bool _index_has_first_key = false) : table_options(_table_options), + ioptions(_ioptions), statistics(_statistics), + raw_ucmp(_raw_ucmp), using_zstd(_using_zstd), protection_bytes_per_key(_protection_bytes_per_key), - raw_ucmp(_raw_ucmp), index_value_is_full(_index_value_is_full), index_has_first_key(_index_has_first_key) {} const BlockBasedTableOptions* table_options = nullptr; + const ImmutableOptions* ioptions = nullptr; Statistics* statistics = nullptr; + const Comparator* raw_ucmp = nullptr; + const UncompressionDict* dict = nullptr; + uint32_t format_version; bool using_zstd = false; uint8_t protection_bytes_per_key = 0; - const Comparator* raw_ucmp = nullptr; bool index_value_is_full; bool index_has_first_key; @@ -95,9 +99,24 @@ struct BlockCreateContext : public Cache::CreateContext { template inline void Create(std::unique_ptr* parsed_out, size_t* charge_out, const Slice& data, - MemoryAllocator* alloc) { - Create(parsed_out, - BlockContents(AllocateAndCopyBlock(data, alloc), data.size())); + CompressionType type, MemoryAllocator* alloc) { + BlockContents uncompressed_block_contents; + if (type != CompressionType::kNoCompression) { + assert(dict != nullptr); + UncompressionContext context(type); + UncompressionInfo info(context, *dict, type); + Status s = UncompressBlockData( + info, data.data(), data.size(), &uncompressed_block_contents, + table_options->format_version, *ioptions, alloc); + if (!s.ok()) { + parsed_out->reset(); + return; + } + } else { + uncompressed_block_contents = + BlockContents(AllocateAndCopyBlock(data, alloc), data.size()); + } + Create(parsed_out, std::move(uncompressed_block_contents)); *charge_out = parsed_out->get()->ApproximateMemoryUsage(); } diff --git a/table/block_based/block_prefetcher.cc b/table/block_based/block_prefetcher.cc index 1734e76d7..b974f9190 100644 --- a/table/block_based/block_prefetcher.cc +++ b/table/block_based/block_prefetcher.cc @@ -15,35 +15,40 @@ namespace ROCKSDB_NAMESPACE { void BlockPrefetcher::PrefetchIfNeeded( const BlockBasedTable::Rep* rep, const BlockHandle& handle, const size_t readahead_size, bool is_for_compaction, - const bool no_sequential_checking, - const Env::IOPriority rate_limiter_priority) { + const bool no_sequential_checking, const ReadOptions& read_options, + const std::function& readaheadsize_cb) { const size_t len = BlockBasedTable::BlockSizeWithTrailer(handle); const size_t offset = handle.offset(); - if (is_for_compaction) { - if (!rep->file->use_direct_io()) { + if (!rep->file->use_direct_io() && compaction_readahead_size_ > 0) { // If FS supports prefetching (readahead_limit_ will be non zero in that // case) and current block exists in prefetch buffer then return. if (offset + len <= readahead_limit_) { return; } - Status s = rep->file->Prefetch(offset, len + compaction_readahead_size_, - rate_limiter_priority); + IOOptions opts; + Status s = rep->file->PrepareIOOptions(read_options, opts); + if (!s.ok()) { + return; + } + s = rep->file->Prefetch(opts, offset, len + compaction_readahead_size_); if (s.ok()) { readahead_limit_ = offset + len + compaction_readahead_size_; return; + } else if (!s.IsNotSupported()) { + return; } } // If FS prefetch is not supported, fall back to use internal prefetch - // buffer. Discarding other return status of Prefetch calls intentionally, - // as we can fallback to reading from disk if Prefetch fails. + // buffer. // // num_file_reads is used by FilePrefetchBuffer only when // implicit_auto_readahead is set. rep->CreateFilePrefetchBufferIfNotExists( compaction_readahead_size_, compaction_readahead_size_, &prefetch_buffer_, /*implicit_auto_readahead=*/false, - /*num_file_reads=*/0, /*num_file_reads_for_auto_readahead=*/0); + /*num_file_reads=*/0, /*num_file_reads_for_auto_readahead=*/0, + /*readaheadsize_cb=*/nullptr); return; } @@ -52,7 +57,8 @@ void BlockPrefetcher::PrefetchIfNeeded( rep->CreateFilePrefetchBufferIfNotExists( readahead_size, readahead_size, &prefetch_buffer_, /*implicit_auto_readahead=*/false, /*num_file_reads=*/0, - /*num_file_reads_for_auto_readahead=*/0); + /*num_file_reads_for_auto_readahead=*/0, readaheadsize_cb, + /*usage=*/FilePrefetchBufferUsage::kUserScanPrefetch); return; } @@ -76,7 +82,8 @@ void BlockPrefetcher::PrefetchIfNeeded( initial_auto_readahead_size_, max_auto_readahead_size, &prefetch_buffer_, /*implicit_auto_readahead=*/true, /*num_file_reads=*/0, - rep->table_options.num_file_reads_for_auto_readahead); + rep->table_options.num_file_reads_for_auto_readahead, readaheadsize_cb, + /*usage=*/FilePrefetchBufferUsage::kUserScanPrefetch); return; } @@ -106,7 +113,8 @@ void BlockPrefetcher::PrefetchIfNeeded( rep->CreateFilePrefetchBufferIfNotExists( initial_auto_readahead_size_, max_auto_readahead_size, &prefetch_buffer_, /*implicit_auto_readahead=*/true, num_file_reads_, - rep->table_options.num_file_reads_for_auto_readahead); + rep->table_options.num_file_reads_for_auto_readahead, readaheadsize_cb, + /*usage=*/FilePrefetchBufferUsage::kUserScanPrefetch); return; } @@ -115,17 +123,20 @@ void BlockPrefetcher::PrefetchIfNeeded( } // If prefetch is not supported, fall back to use internal prefetch buffer. - // Discarding other return status of Prefetch calls intentionally, as - // we can fallback to reading from disk if Prefetch fails. - Status s = rep->file->Prefetch( - handle.offset(), - BlockBasedTable::BlockSizeWithTrailer(handle) + readahead_size_, - rate_limiter_priority); + IOOptions opts; + Status s = rep->file->PrepareIOOptions(read_options, opts); + if (!s.ok()) { + return; + } + s = rep->file->Prefetch( + opts, handle.offset(), + BlockBasedTable::BlockSizeWithTrailer(handle) + readahead_size_); if (s.IsNotSupported()) { rep->CreateFilePrefetchBufferIfNotExists( initial_auto_readahead_size_, max_auto_readahead_size, &prefetch_buffer_, /*implicit_auto_readahead=*/true, num_file_reads_, - rep->table_options.num_file_reads_for_auto_readahead); + rep->table_options.num_file_reads_for_auto_readahead, readaheadsize_cb, + /*usage=*/FilePrefetchBufferUsage::kUserScanPrefetch); return; } diff --git a/table/block_based/block_prefetcher.h b/table/block_based/block_prefetcher.h index 774bf85ac..e46aaf614 100644 --- a/table/block_based/block_prefetcher.h +++ b/table/block_based/block_prefetcher.h @@ -18,11 +18,11 @@ class BlockPrefetcher { readahead_size_(initial_auto_readahead_size), initial_auto_readahead_size_(initial_auto_readahead_size) {} - void PrefetchIfNeeded(const BlockBasedTable::Rep* rep, - const BlockHandle& handle, size_t readahead_size, - bool is_for_compaction, - const bool no_sequential_checking, - Env::IOPriority rate_limiter_priority); + void PrefetchIfNeeded( + const BlockBasedTable::Rep* rep, const BlockHandle& handle, + size_t readahead_size, bool is_for_compaction, + const bool no_sequential_checking, const ReadOptions& read_options, + const std::function& readaheadsize_cb); FilePrefetchBuffer* prefetch_buffer() { return prefetch_buffer_.get(); } void UpdateReadPattern(const uint64_t& offset, const size_t& len) { diff --git a/table/block_based/block_test.cc b/table/block_based/block_test.cc index 3264371c1..b1a855263 100644 --- a/table/block_based/block_test.cc +++ b/table/block_based/block_test.cc @@ -6,9 +6,8 @@ #include "table/block_based/block.h" -#include - #include +#include #include #include #include @@ -848,9 +847,12 @@ TEST_F(BlockPerKVChecksumTest, EmptyBlock) { Options options = Options(); BlockBasedTableOptions tbo; uint8_t protection_bytes_per_key = 8; - BlockCreateContext create_context{ - &tbo, nullptr /* statistics */, false /* using_zstd */, - protection_bytes_per_key, options.comparator}; + BlockCreateContext create_context{&tbo, + nullptr, + nullptr /* statistics */, + false /* using_zstd */, + protection_bytes_per_key, + options.comparator}; create_context.Create(&data_block, std::move(contents)); std::unique_ptr biter{data_block->NewDataIterator( options.comparator, kDisableGlobalSequenceNumber)}; @@ -885,9 +887,12 @@ TEST_F(BlockPerKVChecksumTest, InitializeProtectionInfo) { Options options = Options(); BlockBasedTableOptions tbo; uint8_t protection_bytes_per_key = 8; - BlockCreateContext create_context{ - &tbo, nullptr /* statistics */, false /* using_zstd */, - protection_bytes_per_key, options.comparator}; + BlockCreateContext create_context{&tbo, + nullptr /* ioptions */, + nullptr /* statistics */, + false /* using_zstd */, + protection_bytes_per_key, + options.comparator}; { std::string invalid_content = "1"; @@ -949,14 +954,19 @@ TEST_F(BlockPerKVChecksumTest, ApproximateMemory) { uint8_t protection_bytes_per_key = 8; BlockCreateContext with_checksum_create_context{ &tbo, + nullptr /* ioptions */, nullptr /* statistics */, false /* using_zstd */, protection_bytes_per_key, options.comparator, true /* index_value_is_full */}; - BlockCreateContext create_context{ - &tbo, nullptr /* statistics */, false /* using_zstd */, - 0, options.comparator, true /* index_value_is_full */}; + BlockCreateContext create_context{&tbo, + nullptr /* ioptions */, + nullptr /* statistics */, + false /* using_zstd */, + 0, + options.comparator, + true /* index_value_is_full */}; { std::unique_ptr data_block; @@ -1045,8 +1055,11 @@ class DataBlockKVChecksumTest std::vector &keys, std::vector &values, int num_record) { BlockBasedTableOptions tbo; - BlockCreateContext create_context{&tbo, nullptr /* statistics */, - false /* using_zstd */, GetChecksumLen(), + BlockCreateContext create_context{&tbo, + nullptr /* statistics */, + nullptr /* ioptions */, + false /* using_zstd */, + GetChecksumLen(), Options().comparator}; builder_ = std::make_unique( static_cast(GetRestartInterval()), @@ -1172,6 +1185,7 @@ class IndexBlockKVChecksumTest uint8_t protection_bytes_per_key = GetChecksumLen(); BlockCreateContext create_context{ &tbo, + nullptr /* ioptions */, nullptr /* statistics */, false /* _using_zstd */, protection_bytes_per_key, @@ -1312,9 +1326,12 @@ class MetaIndexBlockKVChecksumTest Options options = Options(); BlockBasedTableOptions tbo; uint8_t protection_bytes_per_key = GetChecksumLen(); - BlockCreateContext create_context{ - &tbo, nullptr /* statistics */, false /* using_zstd */, - protection_bytes_per_key, options.comparator}; + BlockCreateContext create_context{&tbo, + nullptr /* ioptions */, + nullptr /* statistics */, + false /* using_zstd */, + protection_bytes_per_key, + options.comparator}; builder_ = std::make_unique(static_cast(GetRestartInterval())); // add a bunch of records to a block @@ -1344,9 +1361,12 @@ TEST_P(MetaIndexBlockKVChecksumTest, ChecksumConstructionAndVerification) { Options options = Options(); BlockBasedTableOptions tbo; uint8_t protection_bytes_per_key = GetChecksumLen(); - BlockCreateContext create_context{ - &tbo, nullptr /* statistics */, false /* using_zstd */, - protection_bytes_per_key, options.comparator}; + BlockCreateContext create_context{&tbo, + nullptr /* ioptions */, + nullptr /* statistics */, + false /* using_zstd */, + protection_bytes_per_key, + options.comparator}; std::vector num_restart_intervals = {1, 16}; for (const auto num_restart_interval : num_restart_intervals) { const int kNumRecords = num_restart_interval * GetRestartInterval(); @@ -1680,4 +1700,4 @@ int main(int argc, char **argv) { ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); -} \ No newline at end of file +} diff --git a/table/block_based/filter_block_reader_common.cc b/table/block_based/filter_block_reader_common.cc index 32f800db7..e459b00ad 100644 --- a/table/block_based/filter_block_reader_common.cc +++ b/table/block_based/filter_block_reader_common.cc @@ -28,12 +28,12 @@ Status FilterBlockReaderCommon::ReadFilterBlock( const BlockBasedTable::Rep* const rep = table->get_rep(); assert(rep); - const Status s = - table->RetrieveBlock(prefetch_buffer, read_options, rep->filter_handle, - UncompressionDict::GetEmptyDict(), filter_block, - get_context, lookup_context, - /* for_compaction */ false, use_cache, - /* async_read */ false); + const Status s = table->RetrieveBlock( + prefetch_buffer, read_options, rep->filter_handle, + UncompressionDict::GetEmptyDict(), filter_block, get_context, + lookup_context, + /* for_compaction */ false, use_cache, + /* async_read */ false, /* use_block_cache_for_lookup */ true); return s; } diff --git a/table/block_based/filter_policy.cc b/table/block_based/filter_policy.cc index 36f3b16d4..d8e5cd93f 100644 --- a/table/block_based/filter_policy.cc +++ b/table/block_based/filter_policy.cc @@ -10,6 +10,7 @@ #include "rocksdb/filter_policy.h" #include +#include #include #include #include @@ -24,6 +25,7 @@ #include "rocksdb/rocksdb_namespace.h" #include "rocksdb/slice.h" #include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" #include "table/block_based/block_based_table_reader.h" #include "table/block_based/filter_policy_internal.h" #include "table/block_based/full_filter_block.h" @@ -70,7 +72,7 @@ class XXPH3FilterBitsBuilder : public BuiltinFilterBitsBuilder { detect_filter_construct_corruption_( detect_filter_construct_corruption) {} - ~XXPH3FilterBitsBuilder() override {} + ~XXPH3FilterBitsBuilder() override = default; virtual void AddKey(const Slice& key) override { uint64_t hash = GetSliceHash64(key); @@ -319,7 +321,7 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder { FastLocalBloomBitsBuilder(const FastLocalBloomBitsBuilder&) = delete; void operator=(const FastLocalBloomBitsBuilder&) = delete; - ~FastLocalBloomBitsBuilder() override {} + ~FastLocalBloomBitsBuilder() override = default; using FilterBitsBuilder::Finish; @@ -523,7 +525,7 @@ class FastLocalBloomBitsReader : public BuiltinFilterBitsReader { FastLocalBloomBitsReader(const FastLocalBloomBitsReader&) = delete; void operator=(const FastLocalBloomBitsReader&) = delete; - ~FastLocalBloomBitsReader() override {} + ~FastLocalBloomBitsReader() override = default; bool MayMatch(const Slice& key) override { uint64_t h = GetSliceHash64(key); @@ -604,7 +606,7 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder { Standard128RibbonBitsBuilder(const Standard128RibbonBitsBuilder&) = delete; void operator=(const Standard128RibbonBitsBuilder&) = delete; - ~Standard128RibbonBitsBuilder() override {} + ~Standard128RibbonBitsBuilder() override = default; using FilterBitsBuilder::Finish; @@ -965,7 +967,7 @@ class Standard128RibbonBitsReader : public BuiltinFilterBitsReader { Standard128RibbonBitsReader(const Standard128RibbonBitsReader&) = delete; void operator=(const Standard128RibbonBitsReader&) = delete; - ~Standard128RibbonBitsReader() override {} + ~Standard128RibbonBitsReader() override = default; bool MayMatch(const Slice& key) override { uint64_t h = GetSliceHash64(key); @@ -1068,7 +1070,7 @@ LegacyBloomBitsBuilder::LegacyBloomBitsBuilder(const int bits_per_key, assert(bits_per_key_); } -LegacyBloomBitsBuilder::~LegacyBloomBitsBuilder() {} +LegacyBloomBitsBuilder::~LegacyBloomBitsBuilder() = default; void LegacyBloomBitsBuilder::AddKey(const Slice& key) { uint32_t hash = BloomHash(key); @@ -1218,7 +1220,7 @@ class LegacyBloomBitsReader : public BuiltinFilterBitsReader { LegacyBloomBitsReader(const LegacyBloomBitsReader&) = delete; void operator=(const LegacyBloomBitsReader&) = delete; - ~LegacyBloomBitsReader() override {} + ~LegacyBloomBitsReader() override = default; // "contents" contains the data built by a preceding call to // FilterBitsBuilder::Finish. MayMatch must return true if the key was @@ -1357,7 +1359,7 @@ BloomLikeFilterPolicy::BloomLikeFilterPolicy(double bits_per_key) whole_bits_per_key_ = (millibits_per_key_ + 500) / 1000; } -BloomLikeFilterPolicy::~BloomLikeFilterPolicy() {} +BloomLikeFilterPolicy::~BloomLikeFilterPolicy() = default; const char* BloomLikeFilterPolicy::kClassName() { return "rocksdb.internal.BloomLikeFilter"; } @@ -1730,7 +1732,15 @@ const FilterPolicy* NewBloomFilterPolicy(double bits_per_key, RibbonFilterPolicy::RibbonFilterPolicy(double bloom_equivalent_bits_per_key, int bloom_before_level) : BloomLikeFilterPolicy(bloom_equivalent_bits_per_key), - bloom_before_level_(bloom_before_level) {} + bloom_before_level_(bloom_before_level) { + static const std::unordered_map type_info = { + {"bloom_before_level", + {offsetof(class RibbonFilterPolicy, bloom_before_level_), + OptionType::kAtomicInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + }; + RegisterOptions(this, &type_info); +} FilterBitsBuilder* RibbonFilterPolicy::GetBuilderWithContext( const FilterBuildingContext& context) const { @@ -1738,31 +1748,38 @@ FilterBitsBuilder* RibbonFilterPolicy::GetBuilderWithContext( // "No filter" special case return nullptr; } - // Treat unknown same as bottommost - int levelish = INT_MAX; - - switch (context.compaction_style) { - case kCompactionStyleLevel: - case kCompactionStyleUniversal: { - if (context.reason == TableFileCreationReason::kFlush) { - // Treat flush as level -1 - assert(context.level_at_creation == 0); - levelish = -1; - } else if (context.level_at_creation == -1) { - // Unknown level - assert(levelish == INT_MAX); - } else { - levelish = context.level_at_creation; + // Treat unknown same as bottommost, INT_MAX - 1. + // INT_MAX is reserved for "always use Bloom". + int levelish = INT_MAX - 1; + + int bloom_before_level = bloom_before_level_.load(std::memory_order_relaxed); + if (bloom_before_level < INT_MAX) { + switch (context.compaction_style) { + case kCompactionStyleLevel: + case kCompactionStyleUniversal: { + if (context.reason == TableFileCreationReason::kFlush) { + // Treat flush as level -1 + assert(context.level_at_creation == 0); + levelish = -1; + } else if (context.level_at_creation == -1) { + // Unknown level + assert(levelish == INT_MAX - 1); + } else { + levelish = context.level_at_creation; + } + break; } - break; + case kCompactionStyleFIFO: + case kCompactionStyleNone: + // Treat as bottommost + assert(levelish == INT_MAX - 1); + break; } - case kCompactionStyleFIFO: - case kCompactionStyleNone: - // Treat as bottommost - assert(levelish == INT_MAX); - break; + } else { + // INT_MAX == always Bloom + assert(levelish < bloom_before_level); } - if (levelish < bloom_before_level_) { + if (levelish < bloom_before_level) { return GetFastLocalBloomBuilderWithContext(context); } else { return GetStandard128RibbonBuilderWithContext(context); @@ -1771,14 +1788,15 @@ FilterBitsBuilder* RibbonFilterPolicy::GetBuilderWithContext( const char* RibbonFilterPolicy::kClassName() { return "ribbonfilter"; } const char* RibbonFilterPolicy::kNickName() { return "rocksdb.RibbonFilter"; } +const char* RibbonFilterPolicy::kName() { return "RibbonFilterPolicy"; } std::string RibbonFilterPolicy::GetId() const { return BloomLikeFilterPolicy::GetId() + ":" + - std::to_string(bloom_before_level_); + std::to_string(bloom_before_level_.load(std::memory_order_acquire)); } -const FilterPolicy* NewRibbonFilterPolicy(double bloom_equivalent_bits_per_key, - int bloom_before_level) { +FilterPolicy* NewRibbonFilterPolicy(double bloom_equivalent_bits_per_key, + int bloom_before_level) { return new RibbonFilterPolicy(bloom_equivalent_bits_per_key, bloom_before_level); } @@ -1787,7 +1805,7 @@ FilterBuildingContext::FilterBuildingContext( const BlockBasedTableOptions& _table_options) : table_options(_table_options) {} -FilterPolicy::~FilterPolicy() {} +FilterPolicy::~FilterPolicy() = default; std::shared_ptr BloomLikeFilterPolicy::Create( const std::string& name, double bits_per_key) { diff --git a/table/block_based/filter_policy_internal.h b/table/block_based/filter_policy_internal.h index 9bc3a2482..3919c8c6d 100644 --- a/table/block_based/filter_policy_internal.h +++ b/table/block_based/filter_policy_internal.h @@ -290,10 +290,11 @@ class RibbonFilterPolicy : public BloomLikeFilterPolicy { const char* Name() const override { return kClassName(); } static const char* kNickName(); const char* NickName() const override { return kNickName(); } + static const char* kName(); std::string GetId() const override; private: - const int bloom_before_level_; + std::atomic bloom_before_level_; }; // For testing only, but always constructable with internal names diff --git a/table/block_based/full_filter_block.cc b/table/block_based/full_filter_block.cc index 60ff7c44f..0d7d9a599 100644 --- a/table/block_based/full_filter_block.cc +++ b/table/block_based/full_filter_block.cc @@ -259,7 +259,7 @@ void FullFilterBlockReader::MayMatch(MultiGetRange* range, bool no_io, } } - filter_bits_reader->MayMatch(num_keys, &keys[0], &may_match[0]); + filter_bits_reader->MayMatch(num_keys, keys.data(), may_match.data()); int i = 0; for (auto iter = filter_range.begin(); iter != filter_range.end(); ++iter) { diff --git a/table/block_based/full_filter_block_test.cc b/table/block_based/full_filter_block_test.cc index 0268b7b27..154c8c090 100644 --- a/table/block_based/full_filter_block_test.cc +++ b/table/block_based/full_filter_block_test.cc @@ -23,7 +23,7 @@ namespace ROCKSDB_NAMESPACE { class TestFilterBitsBuilder : public FilterBitsBuilder { public: - explicit TestFilterBitsBuilder() {} + explicit TestFilterBitsBuilder() = default; // Add Key to filter void AddKey(const Slice& key) override { @@ -197,7 +197,7 @@ class CountUniqueFilterBitsBuilderWrapper : public FilterBitsBuilder { public: explicit CountUniqueFilterBitsBuilderWrapper(FilterBitsBuilder* b) : b_(b) {} - ~CountUniqueFilterBitsBuilderWrapper() override {} + ~CountUniqueFilterBitsBuilderWrapper() override = default; void AddKey(const Slice& key) override { b_->AddKey(key); diff --git a/table/block_based/index_builder.cc b/table/block_based/index_builder.cc index a9e02a287..98d084b34 100644 --- a/table/block_based/index_builder.cc +++ b/table/block_based/index_builder.cc @@ -9,8 +9,7 @@ #include "table/block_based/index_builder.h" -#include - +#include #include #include #include diff --git a/table/block_based/index_builder.h b/table/block_based/index_builder.h index 56935546d..be690d799 100644 --- a/table/block_based/index_builder.h +++ b/table/block_based/index_builder.h @@ -35,7 +35,7 @@ class IndexBuilder { public: static IndexBuilder* CreateIndexBuilder( BlockBasedTableOptions::IndexType index_type, - const ROCKSDB_NAMESPACE::InternalKeyComparator* comparator, + const InternalKeyComparator* comparator, const InternalKeySliceTransform* int_key_slice_transform, bool use_value_delta_encoding, const BlockBasedTableOptions& table_opt, size_t ts_sz, bool persist_user_defined_timestamps); @@ -106,6 +106,25 @@ class IndexBuilder { virtual bool seperator_is_key_plus_seq() { return true; } protected: + // Given the last key in current block and the first key in the next block, + // return true if internal key should be used as separator, false if user key + // can be used as separator. + inline bool ShouldUseKeyPlusSeqAsSeparator( + const Slice& last_key_in_current_block, + const Slice& first_key_in_next_block) { + Slice l_user_key = ExtractUserKey(last_key_in_current_block); + Slice r_user_key = ExtractUserKey(first_key_in_next_block); + // If user defined timestamps are not persisted. All the user keys will + // act like they have minimal timestamp. Only having user key is not + // sufficient, even if they are different user keys for now, they have to be + // different user keys without the timestamp part. + return persist_user_defined_timestamps_ + ? comparator_->user_comparator()->Compare(l_user_key, + r_user_key) == 0 + : comparator_->user_comparator()->CompareWithoutTimestamp( + l_user_key, r_user_key) == 0; + } + const InternalKeyComparator* comparator_; // Size of user-defined timestamp in bytes. size_t ts_sz_; @@ -173,9 +192,8 @@ class ShortenedIndexBuilder : public IndexBuilder { *first_key_in_next_block); } if (!seperator_is_key_plus_seq_ && - comparator_->user_comparator()->Compare( - ExtractUserKey(*last_key_in_current_block), - ExtractUserKey(*first_key_in_next_block)) == 0) { + ShouldUseKeyPlusSeqAsSeparator(*last_key_in_current_block, + *first_key_in_next_block)) { seperator_is_key_plus_seq_ = true; } } else { @@ -414,9 +432,9 @@ class HashIndexBuilder : public IndexBuilder { class PartitionedIndexBuilder : public IndexBuilder { public: static PartitionedIndexBuilder* CreateIndexBuilder( - const ROCKSDB_NAMESPACE::InternalKeyComparator* comparator, - bool use_value_delta_encoding, const BlockBasedTableOptions& table_opt, - size_t ts_sz, bool persist_user_defined_timestamps); + const InternalKeyComparator* comparator, bool use_value_delta_encoding, + const BlockBasedTableOptions& table_opt, size_t ts_sz, + bool persist_user_defined_timestamps); PartitionedIndexBuilder(const InternalKeyComparator* comparator, const BlockBasedTableOptions& table_opt, diff --git a/table/block_based/index_reader_common.cc b/table/block_based/index_reader_common.cc index 828200299..a1b05c2d6 100644 --- a/table/block_based/index_reader_common.cc +++ b/table/block_based/index_reader_common.cc @@ -26,10 +26,10 @@ Status BlockBasedTable::IndexReaderCommon::ReadIndexBlock( assert(rep != nullptr); const Status s = table->RetrieveBlock( - prefetch_buffer, read_options, rep->footer.index_handle(), + prefetch_buffer, read_options, rep->index_handle, UncompressionDict::GetEmptyDict(), &index_block->As(), get_context, lookup_context, /* for_compaction */ false, use_cache, - /* async_read */ false); + /* async_read */ false, /* use_block_cache_for_lookup */ true); return s; } diff --git a/table/block_based/partitioned_filter_block.cc b/table/block_based/partitioned_filter_block.cc index faddaeb1e..47ac98b9c 100644 --- a/table/block_based/partitioned_filter_block.cc +++ b/table/block_based/partitioned_filter_block.cc @@ -317,12 +317,12 @@ Status PartitionedFilterBlockReader::GetFilterPartitionBlock( read_options.read_tier = kBlockCacheTier; } - const Status s = - table()->RetrieveBlock(prefetch_buffer, read_options, fltr_blk_handle, - UncompressionDict::GetEmptyDict(), filter_block, - get_context, lookup_context, - /* for_compaction */ false, /* use_cache */ true, - /* async_read */ false); + const Status s = table()->RetrieveBlock( + prefetch_buffer, read_options, fltr_blk_handle, + UncompressionDict::GetEmptyDict(), filter_block, get_context, + lookup_context, + /* for_compaction */ false, /* use_cache */ true, + /* async_read */ false, /* use_block_cache_for_lookup */ true); return s; } @@ -497,14 +497,15 @@ Status PartitionedFilterBlockReader::CacheDependencies( tail_prefetch_buffer->GetPrefetchOffset() > prefetch_off) { rep->CreateFilePrefetchBuffer( 0, 0, &prefetch_buffer, false /* Implicit autoreadahead */, - 0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/); + 0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/, + /*readaheadsize_cb*/ nullptr, + /*usage=*/FilePrefetchBufferUsage::kUnknown); IOOptions opts; s = rep->file->PrepareIOOptions(ro, opts); if (s.ok()) { s = prefetch_buffer->Prefetch(opts, rep->file.get(), prefetch_off, - static_cast(prefetch_len), - ro.rate_limiter_priority); + static_cast(prefetch_len)); } if (!s.ok()) { return s; @@ -521,7 +522,8 @@ Status PartitionedFilterBlockReader::CacheDependencies( prefetch_buffer ? prefetch_buffer.get() : tail_prefetch_buffer, ro, handle, UncompressionDict::GetEmptyDict(), /* for_compaction */ false, &block, nullptr /* get_context */, - &lookup_context, nullptr /* contents */, false); + &lookup_context, nullptr /* contents */, false, + /* use_block_cache_for_lookup */ true); if (!s.ok()) { return s; } diff --git a/table/block_based/partitioned_filter_block_test.cc b/table/block_based/partitioned_filter_block_test.cc index 1d6e2fced..50bb77975 100644 --- a/table/block_based/partitioned_filter_block_test.cc +++ b/table/block_based/partitioned_filter_block_test.cc @@ -87,7 +87,7 @@ class PartitionedFilterBlockTest table_options_.index_block_restart_interval = 3; } - ~PartitionedFilterBlockTest() override {} + ~PartitionedFilterBlockTest() override = default; static constexpr int kKeyNum = 4; static constexpr int kMissingKeyNum = 2; @@ -200,7 +200,7 @@ class PartitionedFilterBlockTest // Querying added keys const bool no_io = true; std::vector keys = PrepareKeys(keys_without_ts, kKeyNum); - for (auto key : keys) { + for (const auto& key : keys) { auto ikey = InternalKey(key, 0, ValueType::kTypeValue); const Slice ikey_slice = Slice(*ikey.rep()); ASSERT_TRUE(reader->KeyMayMatch( @@ -220,7 +220,7 @@ class PartitionedFilterBlockTest // querying missing keys std::vector missing_keys = PrepareKeys(missing_keys_without_ts, kMissingKeyNum); - for (auto key : missing_keys) { + for (const auto& key : missing_keys) { auto ikey = InternalKey(key, 0, ValueType::kTypeValue); const Slice ikey_slice = Slice(*ikey.rep()); if (empty) { @@ -386,7 +386,7 @@ TEST_P(PartitionedFilterBlockTest, SamePrefixInMultipleBlocks) { CutABlock(pib.get(), pkeys[2]); std::unique_ptr reader( NewReader(builder.get(), pib.get())); - for (auto key : pkeys) { + for (const auto& key : pkeys) { auto ikey = InternalKey(key, 0, ValueType::kTypeValue); const Slice ikey_slice = Slice(*ikey.rep()); ASSERT_TRUE(reader->PrefixMayMatch(prefix_extractor->Transform(key), @@ -400,7 +400,7 @@ TEST_P(PartitionedFilterBlockTest, SamePrefixInMultipleBlocks) { "p-key31"}; std::vector pnonkeys = PrepareKeys(pnonkeys_without_ts, 4 /* number_of_keys */); - for (auto key : pnonkeys) { + for (const auto& key : pnonkeys) { auto ikey = InternalKey(key, 0, ValueType::kTypeValue); const Slice ikey_slice = Slice(*ikey.rep()); ASSERT_TRUE(reader->PrefixMayMatch(prefix_extractor->Transform(key), @@ -440,7 +440,7 @@ TEST_P(PartitionedFilterBlockTest, PrefixInWrongPartitionBug) { CutABlock(pib.get(), pkeys[4]); std::unique_ptr reader( NewReader(builder.get(), pib.get())); - for (auto key : pkeys) { + for (const auto& key : pkeys) { auto prefix = prefix_extractor->Transform(key); auto ikey = InternalKey(key, 0, ValueType::kTypeValue); const Slice ikey_slice = Slice(*ikey.rep()); diff --git a/table/block_based/partitioned_index_iterator.cc b/table/block_based/partitioned_index_iterator.cc index b9bc2155a..cc6f70130 100644 --- a/table/block_based/partitioned_index_iterator.cc +++ b/table/block_based/partitioned_index_iterator.cc @@ -91,15 +91,16 @@ void PartitionedIndexIterator::InitPartitionedIndexBlock() { // Enabled from the very first IO when ReadOptions.readahead_size is set. block_prefetcher_.PrefetchIfNeeded( rep, partitioned_index_handle, read_options_.readahead_size, - is_for_compaction, /*no_sequential_checking=*/false, - read_options_.rate_limiter_priority); + is_for_compaction, /*no_sequential_checking=*/false, read_options_, + /*readaheadsize_cb=*/nullptr); Status s; table_->NewDataBlockIterator( read_options_, partitioned_index_handle, &block_iter_, BlockType::kIndex, /*get_context=*/nullptr, &lookup_context_, block_prefetcher_.prefetch_buffer(), - /*for_compaction=*/is_for_compaction, /*async_read=*/false, s); + /*for_compaction=*/is_for_compaction, /*async_read=*/false, s, + /*use_block_cache_for_lookup=*/true); block_iter_points_to_real_block_ = true; // We could check upper bound here but it is complicated to reason about // upper bound in index iterator. On the other than, in large scans, index diff --git a/table/block_based/partitioned_index_reader.cc b/table/block_based/partitioned_index_reader.cc index d1c5591eb..b4a16dd22 100644 --- a/table/block_based/partitioned_index_reader.cc +++ b/table/block_based/partitioned_index_reader.cc @@ -169,14 +169,15 @@ Status PartitionIndexReader::CacheDependencies( tail_prefetch_buffer->GetPrefetchOffset() > prefetch_off) { rep->CreateFilePrefetchBuffer( 0, 0, &prefetch_buffer, false /*Implicit auto readahead*/, - 0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/); + 0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/, + /*readaheadsize_cb*/ nullptr, + /*usage=*/FilePrefetchBufferUsage::kUnknown); IOOptions opts; { Status s = rep->file->PrepareIOOptions(ro, opts); if (s.ok()) { s = prefetch_buffer->Prefetch(opts, rep->file.get(), prefetch_off, - static_cast(prefetch_len), - ro.rate_limiter_priority); + static_cast(prefetch_len)); } if (!s.ok()) { return s; @@ -200,7 +201,7 @@ Status PartitionIndexReader::CacheDependencies( handle, UncompressionDict::GetEmptyDict(), /*for_compaction=*/false, &block.As(), /*get_context=*/nullptr, &lookup_context, /*contents=*/nullptr, - /*async_read=*/false); + /*async_read=*/false, /*use_block_cache_for_lookup=*/true); if (!s.ok()) { return s; diff --git a/table/block_based/reader_common.cc b/table/block_based/reader_common.cc index 0ff43e9b4..7d0c97c71 100644 --- a/table/block_based/reader_common.cc +++ b/table/block_based/reader_common.cc @@ -23,10 +23,14 @@ void ForceReleaseCachedEntry(void* arg, void* h) { } // WART: this is specific to block-based table -Status VerifyBlockChecksum(ChecksumType type, const char* data, +Status VerifyBlockChecksum(const Footer& footer, const char* data, size_t block_size, const std::string& file_name, uint64_t offset) { PERF_TIMER_GUARD(block_checksum_time); + + assert(footer.GetBlockTrailerSize() == 5); + ChecksumType type = footer.checksum_type(); + // After block_size bytes is compression type (1 byte), which is part of // the checksummed section. size_t len = block_size + 1; @@ -34,6 +38,13 @@ Status VerifyBlockChecksum(ChecksumType type, const char* data, uint32_t stored = DecodeFixed32(data + len); uint32_t computed = ComputeBuiltinChecksum(type, data, len); + + // Unapply context to 'stored' rather than apply to 'computed, for people + // who might look for reference crc value in error message + uint32_t modifier = + ChecksumModifierForContext(footer.base_context_checksum(), offset); + stored -= modifier; + if (stored == computed) { return Status::OK(); } else { @@ -43,8 +54,9 @@ Status VerifyBlockChecksum(ChecksumType type, const char* data, computed = crc32c::Unmask(computed); } return Status::Corruption( - "block checksum mismatch: stored = " + std::to_string(stored) + - ", computed = " + std::to_string(computed) + + "block checksum mismatch: stored" + + std::string(modifier ? "(context removed)" : "") + " = " + + std::to_string(stored) + ", computed = " + std::to_string(computed) + ", type = " + std::to_string(type) + " in " + file_name + " offset " + std::to_string(offset) + " size " + std::to_string(block_size)); } diff --git a/table/block_based/reader_common.h b/table/block_based/reader_common.h index 102802afe..08c2a756b 100644 --- a/table/block_based/reader_common.h +++ b/table/block_based/reader_common.h @@ -12,6 +12,8 @@ #include "rocksdb/table.h" namespace ROCKSDB_NAMESPACE { +class Footer; + // Release the cached entry and decrement its ref count. extern void ForceReleaseCachedEntry(void* arg, void* h); @@ -22,12 +24,13 @@ inline MemoryAllocator* GetMemoryAllocator( : nullptr; } -// Assumes block has a trailer as in format.h. file_name and offset provided -// for generating a diagnostic message in returned status. +// Assumes block has a trailer past `data + block_size` as in format.h. +// `file_name` provided for generating diagnostic message in returned status. +// `offset` might be required for proper verification (also used for message). // // Returns Status::OK() on checksum match, or Status::Corruption() on checksum // mismatch. -extern Status VerifyBlockChecksum(ChecksumType type, const char* data, +extern Status VerifyBlockChecksum(const Footer& footer, const char* data, size_t block_size, const std::string& file_name, uint64_t offset); diff --git a/table/block_based/uncompression_dict_reader.cc b/table/block_based/uncompression_dict_reader.cc index 4ac442b6b..3656b35d5 100644 --- a/table/block_based/uncompression_dict_reader.cc +++ b/table/block_based/uncompression_dict_reader.cc @@ -63,7 +63,7 @@ Status UncompressionDictReader::ReadUncompressionDictionary( UncompressionDict::GetEmptyDict(), uncompression_dict, get_context, lookup_context, /* for_compaction */ false, use_cache, - /* async_read */ false); + /* async_read */ false, /* use_block_cache_for_lookup */ true); if (!s.ok()) { ROCKS_LOG_WARN( diff --git a/table/block_fetcher.cc b/table/block_fetcher.cc index b2fa6d4b5..257a1a42e 100644 --- a/table/block_fetcher.cc +++ b/table/block_fetcher.cc @@ -33,9 +33,9 @@ inline void BlockFetcher::ProcessTrailerIfPresent() { if (footer_.GetBlockTrailerSize() > 0) { assert(footer_.GetBlockTrailerSize() == BlockBasedTable::kBlockTrailerSize); if (read_options_.verify_checksums) { - io_status_ = status_to_io_status(VerifyBlockChecksum( - footer_.checksum_type(), slice_.data(), block_size_, - file_->file_name(), handle_.offset())); + io_status_ = status_to_io_status( + VerifyBlockChecksum(footer_, slice_.data(), block_size_, + file_->file_name(), handle_.offset())); RecordTick(ioptions_.stats, BLOCK_CHECKSUM_COMPUTE_COUNT); if (!io_status_.ok()) { assert(io_status_.IsCorruption()); @@ -80,11 +80,11 @@ inline bool BlockFetcher::TryGetFromPrefetchBuffer() { if (read_options_.async_io && !for_compaction_) { read_from_prefetch_buffer = prefetch_buffer_->TryReadFromCacheAsync( opts, file_, handle_.offset(), block_size_with_trailer_, &slice_, - &io_s, read_options_.rate_limiter_priority); + &io_s); } else { read_from_prefetch_buffer = prefetch_buffer_->TryReadFromCache( opts, file_, handle_.offset(), block_size_with_trailer_, &slice_, - &io_s, read_options_.rate_limiter_priority, for_compaction_); + &io_s, for_compaction_); } if (read_from_prefetch_buffer) { ProcessTrailerIfPresent(); @@ -259,18 +259,18 @@ IOStatus BlockFetcher::ReadBlockContents() { if (file_->use_direct_io()) { PERF_TIMER_GUARD(block_read_time); PERF_CPU_TIMER_GUARD(block_read_cpu_time, nullptr); - io_status_ = file_->Read( - opts, handle_.offset(), block_size_with_trailer_, &slice_, nullptr, - &direct_io_buf_, read_options_.rate_limiter_priority); + io_status_ = + file_->Read(opts, handle_.offset(), block_size_with_trailer_, + &slice_, nullptr, &direct_io_buf_); PERF_COUNTER_ADD(block_read_count, 1); used_buf_ = const_cast(slice_.data()); } else { PrepareBufferForBlockFromFile(); PERF_TIMER_GUARD(block_read_time); PERF_CPU_TIMER_GUARD(block_read_cpu_time, nullptr); - io_status_ = file_->Read(opts, handle_.offset(), - block_size_with_trailer_, &slice_, used_buf_, - nullptr, read_options_.rate_limiter_priority); + io_status_ = + file_->Read(opts, handle_.offset(), block_size_with_trailer_, + &slice_, used_buf_, nullptr); PERF_COUNTER_ADD(block_read_count, 1); #ifndef NDEBUG if (slice_.data() == &stack_buf_[0]) { @@ -336,9 +336,11 @@ IOStatus BlockFetcher::ReadBlockContents() { #ifndef NDEBUG num_heap_buf_memcpy_++; #endif - compression_type_ = kNoCompression; + // Save the compressed block without trailer + slice_ = Slice(slice_.data(), block_size_); } else { GetBlockContents(); + slice_ = Slice(); } InsertUncompressedBlockToPersistentCacheIfNeeded(); @@ -387,7 +389,6 @@ IOStatus BlockFetcher::ReadAsyncBlockContents() { #ifndef NDEBUG num_heap_buf_memcpy_++; #endif - compression_type_ = kNoCompression; } else { GetBlockContents(); } diff --git a/table/block_fetcher.h b/table/block_fetcher.h index da6c352d0..e5a51e3eb 100644 --- a/table/block_fetcher.h +++ b/table/block_fetcher.h @@ -79,6 +79,10 @@ class BlockFetcher { inline size_t GetBlockSizeWithTrailer() const { return block_size_with_trailer_; } + inline Slice& GetCompressedBlock() { + assert(compression_type_ != kNoCompression); + return slice_; + } #ifndef NDEBUG int TEST_GetNumStackBufMemcpy() const { return num_stack_buf_memcpy_; } diff --git a/table/block_fetcher_test.cc b/table/block_fetcher_test.cc index c2f6552cc..61e444e92 100644 --- a/table/block_fetcher_test.cc +++ b/table/block_fetcher_test.cc @@ -107,6 +107,9 @@ class BlockFetcherTest : public testing::Test { Footer footer; ReadFooter(file.get(), &footer); const BlockHandle& index_handle = footer.index_handle(); + // FIXME: index handle will need to come from metaindex for + // format_version >= 6 when that becomes the default + ASSERT_FALSE(index_handle.IsNull()); CompressionType compression_type; FetchBlock(file.get(), index_handle, BlockType::kIndex, @@ -131,7 +134,9 @@ class BlockFetcherTest : public testing::Test { std::array expected_stats_by_mode) { for (CompressionType compression_type : GetSupportedCompressions()) { bool do_compress = compression_type != kNoCompression; - if (compressed != do_compress) continue; + if (compressed != do_compress) { + continue; + } std::string compression_type_str = CompressionTypeToString(compression_type); @@ -296,7 +301,7 @@ class BlockFetcherTest : public testing::Test { MemoryAllocator* heap_buf_allocator, MemoryAllocator* compressed_buf_allocator, BlockContents* contents, MemcpyStats* stats, - CompressionType* compresstion_type) { + CompressionType* compression_type) { ImmutableOptions ioptions(options_); ReadOptions roptions; PersistentCacheOptions persistent_cache_options; @@ -315,7 +320,11 @@ class BlockFetcherTest : public testing::Test { stats->num_compressed_buf_memcpy = fetcher->TEST_GetNumCompressedBufMemcpy(); - *compresstion_type = fetcher->get_compression_type(); + if (do_uncompress) { + *compression_type = kNoCompression; + } else { + *compression_type = fetcher->get_compression_type(); + } } // NOTE: expected_compression_type is the expected compression diff --git a/table/cuckoo/cuckoo_table_builder.cc b/table/cuckoo/cuckoo_table_builder.cc index 7ca72365f..b0596edac 100644 --- a/table/cuckoo/cuckoo_table_builder.cc +++ b/table/cuckoo/cuckoo_table_builder.cc @@ -5,9 +5,8 @@ #include "table/cuckoo/cuckoo_table_builder.h" -#include - #include +#include #include #include #include @@ -403,8 +402,12 @@ Status CuckooTableBuilder::Finish() { } FooterBuilder footer; - footer.Build(kCuckooTableMagicNumber, /* format_version */ 1, offset, - kNoChecksum, meta_index_block_handle); + Status s = footer.Build(kCuckooTableMagicNumber, /* format_version */ 1, + offset, kNoChecksum, meta_index_block_handle); + if (!s.ok()) { + status_ = s; + return status_; + } io_status_ = file_->Append(footer.GetSlice()); status_ = io_status_; return status_; @@ -477,7 +480,7 @@ bool CuckooTableBuilder::MakeSpaceForKey( uint64_t bid = hash_vals[hash_cnt]; (*buckets)[static_cast(bid)].make_space_for_key_call_id = make_space_for_key_call_id; - tree.push_back(CuckooNode(bid, 0, 0)); + tree.emplace_back(bid, 0, 0); } bool null_found = false; uint32_t curr_pos = 0; @@ -503,7 +506,7 @@ bool CuckooTableBuilder::MakeSpaceForKey( } (*buckets)[static_cast(child_bucket_id)] .make_space_for_key_call_id = make_space_for_key_call_id; - tree.push_back(CuckooNode(child_bucket_id, curr_depth + 1, curr_pos)); + tree.emplace_back(child_bucket_id, curr_depth + 1, curr_pos); if ((*buckets)[static_cast(child_bucket_id)].vector_idx == kMaxVectorIdx) { null_found = true; diff --git a/table/cuckoo/cuckoo_table_builder_test.cc b/table/cuckoo/cuckoo_table_builder_test.cc index ceddbf37a..1a0d58c76 100644 --- a/table/cuckoo/cuckoo_table_builder_test.cc +++ b/table/cuckoo/cuckoo_table_builder_test.cc @@ -122,8 +122,7 @@ class CuckooBuilderTest : public testing::Test { for (uint32_t i = 0; i + 1 < table_size + cuckoo_block_size; ++i) { Slice read_slice; ASSERT_OK(file_reader->Read(IOOptions(), i * bucket_size, bucket_size, - &read_slice, nullptr, nullptr, - Env::IO_TOTAL /* rate_limiter_priority */)); + &read_slice, nullptr, nullptr)); size_t key_idx = std::find(expected_locations.begin(), expected_locations.end(), i) - expected_locations.begin(); diff --git a/table/cuckoo/cuckoo_table_reader.cc b/table/cuckoo/cuckoo_table_reader.cc index d64761962..54ae6266e 100644 --- a/table/cuckoo/cuckoo_table_reader.cc +++ b/table/cuckoo/cuckoo_table_reader.cc @@ -43,7 +43,7 @@ CuckooTableReader::CuckooTableReader( identity_as_first_hash_(false), use_module_hash_(false), num_hash_func_(0), - unused_key_(""), + key_length_(0), user_key_length_(0), value_length_(0), @@ -144,9 +144,8 @@ CuckooTableReader::CuckooTableReader( *reinterpret_cast(cuckoo_block_size->second.data()); cuckoo_block_bytes_minus_one_ = cuckoo_block_size_ * bucket_length_ - 1; // TODO: rate limit reads of whole cuckoo tables. - status_ = - file_->Read(IOOptions(), 0, static_cast(file_size), &file_data_, - nullptr, nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + status_ = file_->Read(IOOptions(), 0, static_cast(file_size), + &file_data_, nullptr, nullptr); } Status CuckooTableReader::Get(const ReadOptions& /*readOptions*/, @@ -183,7 +182,9 @@ Status CuckooTableReader::Get(const ReadOptions& /*readOptions*/, ParsedInternalKey found_ikey; Status s = ParseInternalKey(full_key, &found_ikey, false /* log_err_key */); // TODO - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } bool dont_care __attribute__((__unused__)); get_context->SaveValue(found_ikey, value, &dont_care); } @@ -214,7 +215,7 @@ class CuckooTableIterator : public InternalIterator { // No copying allowed CuckooTableIterator(const CuckooTableIterator&) = delete; void operator=(const Iterator&) = delete; - ~CuckooTableIterator() override {} + ~CuckooTableIterator() override = default; bool Valid() const override; void SeekToFirst() override; void SeekToLast() override; diff --git a/table/cuckoo/cuckoo_table_reader_test.cc b/table/cuckoo/cuckoo_table_reader_test.cc index e83baa107..d829b3630 100644 --- a/table/cuckoo/cuckoo_table_reader_test.cc +++ b/table/cuckoo/cuckoo_table_reader_test.cc @@ -249,7 +249,7 @@ TEST_F(CuckooReaderTest, WhenKeyExistsWithUint64Comparator) { fname = test::PerThreadDBPath("CuckooReaderUint64_WhenKeyExists"); for (uint64_t i = 0; i < num_items; i++) { user_keys[i].resize(8); - memcpy(&user_keys[i][0], static_cast(&i), 8); + memcpy(user_keys[i].data(), static_cast(&i), 8); ParsedInternalKey ikey(user_keys[i], i + 1000, kTypeValue); AppendInternalKey(&keys[i], ikey); values[i] = "value" + NumToStr(i); @@ -296,7 +296,7 @@ TEST_F(CuckooReaderTest, CheckIteratorUint64) { fname = test::PerThreadDBPath("CuckooReader_CheckIterator"); for (uint64_t i = 0; i < num_items; i++) { user_keys[i].resize(8); - memcpy(&user_keys[i][0], static_cast(&i), 8); + memcpy(user_keys[i].data(), static_cast(&i), 8); ParsedInternalKey ikey(user_keys[i], 1000, kTypeValue); AppendInternalKey(&keys[i], ikey); values[i] = "value" + NumToStr(i); @@ -425,7 +425,7 @@ void WriteFile(const std::vector& keys, const uint64_t num, ASSERT_OK(builder.status()); for (uint64_t key_idx = 0; key_idx < num; ++key_idx) { // Value is just a part of key. - builder.Add(Slice(keys[key_idx]), Slice(&keys[key_idx][0], 4)); + builder.Add(Slice(keys[key_idx]), Slice(keys[key_idx].data(), 4)); ASSERT_EQ(builder.NumEntries(), key_idx + 1); ASSERT_OK(builder.status()); } @@ -454,7 +454,7 @@ void WriteFile(const std::vector& keys, const uint64_t num, value.Reset(); value.clear(); ASSERT_OK(reader.Get(r_options, Slice(keys[i]), &get_context, nullptr)); - ASSERT_TRUE(Slice(keys[i]) == Slice(&keys[i][0], 4)); + ASSERT_TRUE(Slice(keys[i]) == Slice(keys[i].data(), 4)); } } diff --git a/table/format.cc b/table/format.cc index dc077bd45..939b7dcd6 100644 --- a/table/format.cc +++ b/table/format.cc @@ -10,6 +10,7 @@ #include "table/format.h" #include +#include #include #include "block_fetcher.h" @@ -18,12 +19,14 @@ #include "monitoring/perf_context_imp.h" #include "monitoring/statistics_impl.h" #include "options/options_helper.h" +#include "port/likely.h" #include "rocksdb/env.h" #include "rocksdb/options.h" #include "rocksdb/table.h" #include "table/block_based/block.h" #include "table/block_based/block_based_table_reader.h" #include "table/persistent_cache_helper.h" +#include "unique_id_impl.h" #include "util/cast_util.h" #include "util/coding.h" #include "util/compression.h" @@ -195,25 +198,41 @@ inline uint8_t BlockTrailerSizeForMagicNumber(uint64_t magic_number) { // -> format_version >= 1 // checksum type (char, 1 byte) // * Part2 +// -> format_version <= 5 // metaindex handle (varint64 offset, varint64 size) // index handle (varint64 offset, varint64 size) // for part2 size = 2 * BlockHandle::kMaxEncodedLength = 40 +// - This padding is unchecked/ignored +// -> format_version >= 6 +// extended magic number (4 bytes) = 0x3e 0x00 0x7a 0x00 +// - Also surely invalid (size 0) handles if interpreted as older version +// - (Helps ensure a corrupted format_version doesn't get us far with no +// footer checksum.) +// footer_checksum (uint32LE, 4 bytes) +// - Checksum of above checksum type of whole footer, with this field +// set to all zeros. +// base_context_checksum (uint32LE, 4 bytes) +// metaindex block size (uint32LE, 4 bytes) +// - Assumed to be immediately before footer, < 4GB +// (24 bytes, reserved for future use) +// - Brings part2 size also to 40 bytes +// - Checked that last eight bytes == 0, so reserved for a future +// incompatible feature (but under format_version=6) // * Part3 // -> format_version == 0 (inferred from legacy magic number) // legacy magic number (8 bytes) // -> format_version >= 1 (inferred from NOT legacy magic number) // format_version (uint32LE, 4 bytes), also called "footer version" // newer magic number (8 bytes) - +const std::array kExtendedMagic{{0x3e, 0x00, 0x7a, 0x00}}; constexpr size_t kFooterPart2Size = 2 * BlockHandle::kMaxEncodedLength; } // namespace -void FooterBuilder::Build(uint64_t magic_number, uint32_t format_version, - uint64_t footer_offset, ChecksumType checksum_type, - const BlockHandle& metaindex_handle, - const BlockHandle& index_handle) { - (void)footer_offset; // Future use - +Status FooterBuilder::Build(uint64_t magic_number, uint32_t format_version, + uint64_t footer_offset, ChecksumType checksum_type, + const BlockHandle& metaindex_handle, + const BlockHandle& index_handle, + uint32_t base_context_checksum) { assert(magic_number != Footer::kNullTableMagicNumber); assert(IsSupportedFormatVersion(format_version)); @@ -249,19 +268,71 @@ void FooterBuilder::Build(uint64_t magic_number, uint32_t format_version, assert(cur + 8 == slice_.data() + slice_.size()); } - { + if (format_version >= 6) { + if (BlockTrailerSizeForMagicNumber(magic_number) != 0) { + // base context checksum required for table formats with block checksums + assert(base_context_checksum != 0); + assert(ChecksumModifierForContext(base_context_checksum, 0) != 0); + } else { + // base context checksum not used + assert(base_context_checksum == 0); + assert(ChecksumModifierForContext(base_context_checksum, 0) == 0); + } + + // Start populating Part 2 + char* cur = data_.data() + /* part 1 size */ 1; + // Set extended magic of part2 + std::copy(kExtendedMagic.begin(), kExtendedMagic.end(), cur); + cur += kExtendedMagic.size(); + // Fill checksum data with zeros (for later computing checksum) + char* checksum_data = cur; + EncodeFixed32(cur, 0); + cur += 4; + // Save base context checksum + EncodeFixed32(cur, base_context_checksum); + cur += 4; + // Compute and save metaindex size + uint32_t metaindex_size = static_cast(metaindex_handle.size()); + if (metaindex_size != metaindex_handle.size()) { + return Status::NotSupported("Metaindex block size > 4GB"); + } + // Metaindex must be adjacent to footer + assert(metaindex_size == 0 || + metaindex_handle.offset() + metaindex_handle.size() == + footer_offset - BlockTrailerSizeForMagicNumber(magic_number)); + EncodeFixed32(cur, metaindex_size); + cur += 4; + + // Zero pad remainder (for future use) + std::fill_n(cur, 24U, char{0}); + assert(cur + 24 == part3); + + // Compute checksum, add context + uint32_t checksum = ComputeBuiltinChecksum( + checksum_type, data_.data(), Footer::kNewVersionsEncodedLength); + checksum += + ChecksumModifierForContext(base_context_checksum, footer_offset); + // Store it + EncodeFixed32(checksum_data, checksum); + } else { + // Base context checksum not used + assert(!FormatVersionUsesContextChecksum(format_version)); + // Should be left empty + assert(base_context_checksum == 0); + assert(ChecksumModifierForContext(base_context_checksum, 0) == 0); + + // Populate all of part 2 char* cur = part2; cur = metaindex_handle.EncodeTo(cur); cur = index_handle.EncodeTo(cur); // Zero pad remainder std::fill(cur, part3, char{0}); } + return Status::OK(); } Status Footer::DecodeFrom(Slice input, uint64_t input_offset, uint64_t enforce_table_magic_number) { - (void)input_offset; // Future use - // Only decode to unused Footer assert(table_magic_number_ == kNullTableMagicNumber); assert(input != nullptr); @@ -284,6 +355,9 @@ Status Footer::DecodeFrom(Slice input, uint64_t input_offset, block_trailer_size_ = BlockTrailerSizeForMagicNumber(magic); // Parse Part3 + const char* part3_ptr = magic_ptr; + uint32_t computed_checksum = 0; + uint64_t footer_offset = 0; if (legacy) { // The size is already asserted to be at least kMinEncodedLength // at the beginning of the function @@ -291,37 +365,101 @@ Status Footer::DecodeFrom(Slice input, uint64_t input_offset, format_version_ = 0 /* legacy */; checksum_type_ = kCRC32c; } else { - const char* part3_ptr = magic_ptr - 4; + part3_ptr = magic_ptr - 4; format_version_ = DecodeFixed32(part3_ptr); - if (!IsSupportedFormatVersion(format_version_)) { + if (UNLIKELY(!IsSupportedFormatVersion(format_version_))) { return Status::Corruption("Corrupt or unsupported format_version: " + std::to_string(format_version_)); } // All known format versions >= 1 occupy exactly this many bytes. - if (input.size() < kNewVersionsEncodedLength) { + if (UNLIKELY(input.size() < kNewVersionsEncodedLength)) { return Status::Corruption("Input is too short to be an SST file"); } uint64_t adjustment = input.size() - kNewVersionsEncodedLength; input.remove_prefix(adjustment); + footer_offset = input_offset + adjustment; // Parse Part1 char chksum = input.data()[0]; checksum_type_ = lossless_cast(chksum); - if (!IsSupportedChecksumType(checksum_type())) { + if (UNLIKELY(!IsSupportedChecksumType(checksum_type()))) { return Status::Corruption("Corrupt or unsupported checksum type: " + std::to_string(lossless_cast(chksum))); } + // This is the most convenient place to compute the checksum + if (checksum_type_ != kNoChecksum && format_version_ >= 6) { + std::array copy_without_checksum; + std::copy_n(input.data(), kNewVersionsEncodedLength, + copy_without_checksum.data()); + EncodeFixed32(©_without_checksum[5], 0); // Clear embedded checksum + computed_checksum = + ComputeBuiltinChecksum(checksum_type(), copy_without_checksum.data(), + kNewVersionsEncodedLength); + } // Consume checksum type field input.remove_prefix(1); } // Parse Part2 - Status result = metaindex_handle_.DecodeFrom(&input); - if (result.ok()) { - result = index_handle_.DecodeFrom(&input); + if (format_version_ >= 6) { + Slice ext_magic(input.data(), 4); + if (UNLIKELY(ext_magic.compare(Slice(kExtendedMagic.data(), + kExtendedMagic.size())) != 0)) { + return Status::Corruption("Bad extended magic number: 0x" + + ext_magic.ToString(/*hex*/ true)); + } + input.remove_prefix(4); + uint32_t stored_checksum = 0, metaindex_size = 0; + bool success; + success = GetFixed32(&input, &stored_checksum); + assert(success); + success = GetFixed32(&input, &base_context_checksum_); + assert(success); + if (UNLIKELY(ChecksumModifierForContext(base_context_checksum_, 0) == 0)) { + return Status::Corruption("Invalid base context checksum"); + } + computed_checksum += + ChecksumModifierForContext(base_context_checksum_, footer_offset); + if (UNLIKELY(computed_checksum != stored_checksum)) { + return Status::Corruption("Footer at " + std::to_string(footer_offset) + + " checksum mismatch"); + } + success = GetFixed32(&input, &metaindex_size); + assert(success); + (void)success; + uint64_t metaindex_end = footer_offset - GetBlockTrailerSize(); + metaindex_handle_ = + BlockHandle(metaindex_end - metaindex_size, metaindex_size); + + // Mark unpopulated + index_handle_ = BlockHandle::NullBlockHandle(); + + // 16 bytes of unchecked reserved padding + input.remove_prefix(16U); + + // 8 bytes of checked reserved padding (expected to be zero unless using a + // future feature). + uint64_t reserved = 0; + success = GetFixed64(&input, &reserved); + assert(success); + if (UNLIKELY(reserved != 0)) { + return Status::NotSupported( + "File uses a future feature not supported in this version"); + } + // End of part 2 + assert(input.data() == part3_ptr); + } else { + // format_version_ < 6 + Status result = metaindex_handle_.DecodeFrom(&input); + if (result.ok()) { + result = index_handle_.DecodeFrom(&input); + } + if (!result.ok()) { + return result; + } + // Padding in part2 is ignored } - return result; - // Padding in part2 is ignored + return Status::OK(); } std::string Footer::ToString() const { @@ -371,20 +509,20 @@ Status ReadFooterFromFile(const IOOptions& opts, RandomAccessFileReader* file, // need to pass a timeout at that point // TODO: rate limit footer reads. if (prefetch_buffer == nullptr || - !prefetch_buffer->TryReadFromCache( - opts, file, read_offset, Footer::kMaxEncodedLength, &footer_input, - nullptr, opts.rate_limiter_priority)) { + !prefetch_buffer->TryReadFromCache(opts, file, read_offset, + Footer::kMaxEncodedLength, + &footer_input, nullptr)) { if (file->use_direct_io()) { s = file->Read(opts, read_offset, Footer::kMaxEncodedLength, - &footer_input, nullptr, &internal_buf, - opts.rate_limiter_priority); + &footer_input, nullptr, &internal_buf); } else { footer_buf.reserve(Footer::kMaxEncodedLength); s = file->Read(opts, read_offset, Footer::kMaxEncodedLength, - &footer_input, &footer_buf[0], nullptr, - opts.rate_limiter_priority); + &footer_input, footer_buf.data(), nullptr); + } + if (!s.ok()) { + return s; } - if (!s.ok()) return s; } // Check that we actually read the whole footer from the file. It may be @@ -509,19 +647,25 @@ Status UncompressBlockData(const UncompressionInfo& uncompression_info, StopWatchNano timer(ioptions.clock, ShouldReportDetailedTime(ioptions.env, ioptions.stats)); size_t uncompressed_size = 0; - CacheAllocationPtr ubuf = - UncompressData(uncompression_info, data, size, &uncompressed_size, - GetCompressFormatForVersion(format_version), allocator); + const char* error_msg = nullptr; + CacheAllocationPtr ubuf = UncompressData( + uncompression_info, data, size, &uncompressed_size, + GetCompressFormatForVersion(format_version), allocator, &error_msg); if (!ubuf) { if (!CompressionTypeSupported(uncompression_info.type())) { - return Status::NotSupported( + ret = Status::NotSupported( "Unsupported compression method for this build", CompressionTypeToString(uncompression_info.type())); } else { - return Status::Corruption( - "Corrupted compressed block contents", - CompressionTypeToString(uncompression_info.type())); + std::ostringstream oss; + oss << "Corrupted compressed block contents"; + if (error_msg) { + oss << ": " << error_msg; + } + ret = Status::Corruption( + oss.str(), CompressionTypeToString(uncompression_info.type())); } + return ret; } *out_contents = BlockContents(std::move(ubuf), uncompressed_size); diff --git a/table/format.h b/table/format.h index c375165bf..73675381e 100644 --- a/table/format.h +++ b/table/format.h @@ -111,6 +111,40 @@ struct IndexValue { std::string ToString(bool hex, bool have_first_key) const; }; +// Given a file's base_context_checksum and an offset of a block within that +// file, choose a 32-bit value that is as unique as possible. This value will +// be added to the standard checksum to get a checksum "with context," or can +// be subtracted to "remove" context. Returns zero (no modifier) if feature is +// disabled with base_context_checksum == 0. +inline uint32_t ChecksumModifierForContext(uint32_t base_context_checksum, + uint64_t offset) { + // To disable on base_context_checksum == 0, we could write + // `if (base_context_checksum == 0) return 0;` but benchmarking shows + // measurable performance penalty vs. this: compute the modifier + // unconditionally and use an "all or nothing" bit mask to enable + // or disable. + uint32_t all_or_nothing = uint32_t{0} - (base_context_checksum != 0); + + // Desired properties: + // (call this function f(b, o) where b = base and o = offset) + // 1. Fast + // 2. f(b1, o) == f(b2, o) iff b1 == b2 + // (Perfectly preserve base entropy) + // 3. f(b, o1) == f(b, o2) only if o1 == o2 or |o1-o2| >= 4 billion + // (Guaranteed uniqueness for nearby offsets) + // 3. f(b, o + j * 2**32) == f(b, o + k * 2**32) only if j == k + // (Upper bits matter, and *aligned* misplacement fails check) + // 4. f(b1, o) == f(b2, o + x) then preferably not + // f(b1, o + y) == f(b2, o + x + y) + // (Avoid linearly correlated matches) + // 5. f(b, o) == 0 depends on both b and o + // (No predictable overlap with non-context checksums) + uint32_t modifier = + base_context_checksum ^ (Lower32of64(offset) + Upper32of64(offset)); + + return modifier & all_or_nothing; +} + inline uint32_t GetCompressFormatForVersion(uint32_t format_version) { // As of format_version 2, we encode compressed block with // compress_format_version == 2. Before that, the version is 1. @@ -118,18 +152,27 @@ inline uint32_t GetCompressFormatForVersion(uint32_t format_version) { return format_version >= 2 ? 2 : 1; } -constexpr uint32_t kLatestFormatVersion = 5; +constexpr uint32_t kLatestFormatVersion = 6; inline bool IsSupportedFormatVersion(uint32_t version) { return version <= kLatestFormatVersion; } +// Same as having a unique id in footer. +inline bool FormatVersionUsesContextChecksum(uint32_t version) { + return version >= 6; +} + +inline bool FormatVersionUsesIndexHandleInFooter(uint32_t version) { + return version < 6; +} + // Footer encapsulates the fixed information stored at the tail end of every // SST file. In general, it should only include things that cannot go // elsewhere under the metaindex block. For example, checksum_type is // required for verifying metaindex block checksum (when applicable), but -// index block handle can easily go in metaindex block (possible future). -// See also FooterBuilder below. +// index block handle can easily go in metaindex block. See also FooterBuilder +// below. class Footer { public: // Create empty. Populate using DecodeFrom. @@ -137,7 +180,7 @@ class Footer { // Deserialize a footer (populate fields) from `input` and check for various // corruptions. `input_offset` is the offset within the target file of - // `input` buffer (future use). + // `input` buffer, which is needed for verifying format_version >= 6 footer. // If enforce_table_magic_number != 0, will return corruption if table magic // number is not equal to enforce_table_magic_number. Status DecodeFrom(Slice input, uint64_t input_offset, @@ -152,13 +195,17 @@ class Footer { // BBTO::format_version.) uint32_t format_version() const { return format_version_; } + // See ChecksumModifierForContext() + uint32_t base_context_checksum() const { return base_context_checksum_; } + // Block handle for metaindex block. const BlockHandle& metaindex_handle() const { return metaindex_handle_; } // Block handle for (top-level) index block. + // TODO? remove from this struct and only read on decode for legacy cases const BlockHandle& index_handle() const { return index_handle_; } - // Checksum type used in the file. + // Checksum type used in the file, including footer for format version >= 6. ChecksumType checksum_type() const { return static_cast(checksum_type_); } @@ -198,6 +245,7 @@ class Footer { uint64_t table_magic_number_ = kNullTableMagicNumber; uint32_t format_version_ = kInvalidFormatVersion; + uint32_t base_context_checksum_ = 0; BlockHandle metaindex_handle_; BlockHandle index_handle_; int checksum_type_ = kInvalidChecksumType; @@ -219,11 +267,16 @@ class FooterBuilder { // * footer_offset is the file offset where the footer will be written // (for future use). // * checksum_type is for formats using block checksums. - // * index_handle is optional for some kinds of SST files. - void Build(uint64_t table_magic_number, uint32_t format_version, - uint64_t footer_offset, ChecksumType checksum_type, - const BlockHandle& metaindex_handle, - const BlockHandle& index_handle = BlockHandle::NullBlockHandle()); + // * index_handle is optional for some SST kinds and (for caller convenience) + // ignored when format_version >= 6. (Must be added to metaindex in that + // case.) + // * unique_id must be specified if format_vesion >= 6 and SST uses block + // checksums with context. Otherwise, auto-generated if format_vesion >= 6. + Status Build(uint64_t table_magic_number, uint32_t format_version, + uint64_t footer_offset, ChecksumType checksum_type, + const BlockHandle& metaindex_handle, + const BlockHandle& index_handle = BlockHandle::NullBlockHandle(), + uint32_t base_context_checksum = 0); // After Builder, get a Slice for the serialized Footer, backed by this // FooterBuilder. diff --git a/table/get_context.cc b/table/get_context.cc index 8f5cd75f1..7dafbd7d4 100644 --- a/table/get_context.cc +++ b/table/get_context.cc @@ -19,22 +19,6 @@ namespace ROCKSDB_NAMESPACE { -namespace { - -void appendToReplayLog(std::string* replay_log, ValueType type, Slice value) { - if (replay_log) { - if (replay_log->empty()) { - // Optimization: in the common case of only one operation in the - // log, we allocate the exact amount of space needed. - replay_log->reserve(1 + VarintLength(value.size()) + value.size()); - } - replay_log->push_back(type); - PutLengthPrefixedSlice(replay_log, value); - } -} - -} // namespace - GetContext::GetContext( const Comparator* ucmp, const MergeOperator* merge_operator, Logger* logger, Statistics* statistics, GetState init_state, const Slice& user_key, @@ -88,6 +72,24 @@ GetContext::GetContext(const Comparator* ucmp, seq, _pinned_iters_mgr, callback, is_blob_index, tracing_get_id, blob_fetcher) {} +void GetContext::appendToReplayLog(ValueType type, Slice value, Slice ts) { + if (replay_log_) { + if (replay_log_->empty()) { + // Optimization: in the common case of only one operation in the + // log, we allocate the exact amount of space needed. + replay_log_->reserve(1 + VarintLength(value.size()) + value.size()); + } + replay_log_->push_back(type); + PutLengthPrefixedSlice(replay_log_, value); + + // If cf enables ts, there should always be a ts following each value + if (ucmp_->timestamp_size() > 0) { + assert(ts.size() == ucmp_->timestamp_size()); + PutLengthPrefixedSlice(replay_log_, ts); + } + } +} + // Called from TableCache::Get and Table::Get when file/block in which // key may exist are not there in TableCache/BlockCache respectively. In this // case we can't guarantee that key does not exist and are not permitted to do @@ -102,7 +104,9 @@ void GetContext::MarkKeyMayExist() { void GetContext::SaveValue(const Slice& value, SequenceNumber /*seq*/) { assert(state_ == kNotFound); - appendToReplayLog(replay_log_, kTypeValue, value); + assert(ucmp_->timestamp_size() == 0); + + appendToReplayLog(kTypeValue, value, Slice()); state_ = kFound; if (LIKELY(pinnable_val_ != nullptr)) { @@ -228,8 +232,6 @@ bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, return true; // to continue to the next seq } - appendToReplayLog(replay_log_, parsed_key.type, value); - if (seq_ != nullptr) { // Set the sequence number if it is uninitialized if (*seq_ == kMaxSequenceNumber) { @@ -241,32 +243,37 @@ bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, } size_t ts_sz = ucmp_->timestamp_size(); - if (ts_sz > 0 && timestamp_ != nullptr) { - if (!timestamp_->empty()) { - assert(ts_sz == timestamp_->size()); - // `timestamp` can be set before `SaveValue` is ever called - // when max_covering_tombstone_seq_ was set. - // If this key has a higher sequence number than range tombstone, - // then timestamp should be updated. `ts_from_rangetombstone_` is - // set to false afterwards so that only the key with highest seqno - // updates the timestamp. - if (ts_from_rangetombstone_) { - assert(max_covering_tombstone_seq_); - if (parsed_key.sequence > *max_covering_tombstone_seq_) { - Slice ts = ExtractTimestampFromUserKey(parsed_key.user_key, ts_sz); - timestamp_->assign(ts.data(), ts.size()); - ts_from_rangetombstone_ = false; + Slice ts; + + if (ts_sz > 0) { + // ensure always have ts if cf enables ts. + ts = ExtractTimestampFromUserKey(parsed_key.user_key, ts_sz); + if (timestamp_ != nullptr) { + if (!timestamp_->empty()) { + assert(ts_sz == timestamp_->size()); + // `timestamp` can be set before `SaveValue` is ever called + // when max_covering_tombstone_seq_ was set. + // If this key has a higher sequence number than range tombstone, + // then timestamp should be updated. `ts_from_rangetombstone_` is + // set to false afterwards so that only the key with highest seqno + // updates the timestamp. + if (ts_from_rangetombstone_) { + assert(max_covering_tombstone_seq_); + if (parsed_key.sequence > *max_covering_tombstone_seq_) { + timestamp_->assign(ts.data(), ts.size()); + ts_from_rangetombstone_ = false; + } } } - } - // TODO optimize for small size ts - const std::string kMaxTs(ts_sz, '\xff'); - if (timestamp_->empty() || - ucmp_->CompareTimestamp(*timestamp_, kMaxTs) == 0) { - Slice ts = ExtractTimestampFromUserKey(parsed_key.user_key, ts_sz); - timestamp_->assign(ts.data(), ts.size()); + // TODO optimize for small size ts + const std::string kMaxTs(ts_sz, '\xff'); + if (timestamp_->empty() || + ucmp_->CompareTimestamp(*timestamp_, kMaxTs) == 0) { + timestamp_->assign(ts.data(), ts.size()); + } } } + appendToReplayLog(parsed_key.type, value, ts); auto type = parsed_key.type; // Key matches. Process it @@ -374,7 +381,7 @@ bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, Slice blob_value(pin_val); state_ = kFound; if (do_merge_) { - Merge(&blob_value); + MergeWithPlainBaseValue(blob_value); } else { // It means this function is called as part of DB GetMergeOperands // API and the current value should be part of @@ -385,7 +392,7 @@ bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, state_ = kFound; if (do_merge_) { - MergeWithEntity(value); + MergeWithWideColumnBaseValue(value); } else { // It means this function is called as part of DB GetMergeOperands // API and the current value should be part of @@ -407,7 +414,7 @@ bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, state_ = kFound; if (do_merge_) { - Merge(&value); + MergeWithPlainBaseValue(value); } else { // It means this function is called as part of DB GetMergeOperands // API and the current value should be part of @@ -430,7 +437,7 @@ bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, } else if (kMerge == state_) { state_ = kFound; if (do_merge_) { - Merge(nullptr); + MergeWithNoBaseValue(); } // If do_merge_ = false then the current value shouldn't be part of // merge_context_->operand_list @@ -448,7 +455,7 @@ bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, merge_operator_->ShouldMerge( merge_context_->GetOperandsDirectionBackward())) { state_ = kFound; - Merge(nullptr); + MergeWithNoBaseValue(); return false; } return true; @@ -463,20 +470,9 @@ bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, return false; } -void GetContext::Merge(const Slice* value) { - assert(do_merge_); - assert(!pinnable_val_ || !columns_); - - std::string result; - // `op_failure_scope` (an output parameter) is not provided (set to nullptr) - // since a failure must be propagated regardless of its value. - const Status s = MergeHelper::TimedFullMerge( - merge_operator_, user_key_, value, merge_context_->GetOperands(), &result, - logger_, statistics_, clock_, /* result_operand */ nullptr, - /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - if (!s.ok()) { - if (s.subcode() == Status::SubCode::kMergeOperatorFailed) { +void GetContext::PostprocessMerge(const Status& merge_status) { + if (!merge_status.ok()) { + if (merge_status.subcode() == Status::SubCode::kMergeOperatorFailed) { state_ = kMergeOperatorFailed; } else { state_ = kCorrupt; @@ -485,81 +481,53 @@ void GetContext::Merge(const Slice* value) { } if (LIKELY(pinnable_val_ != nullptr)) { - *(pinnable_val_->GetSelf()) = std::move(result); pinnable_val_->PinSelf(); - return; } - - assert(columns_); - columns_->SetPlainValue(std::move(result)); } -void GetContext::MergeWithEntity(Slice entity) { +void GetContext::MergeWithNoBaseValue() { assert(do_merge_); + assert(pinnable_val_ || columns_); assert(!pinnable_val_ || !columns_); - if (LIKELY(pinnable_val_ != nullptr)) { - Slice value_of_default; - - { - const Status s = WideColumnSerialization::GetValueOfDefaultColumn( - entity, value_of_default); - if (!s.ok()) { - state_ = kCorrupt; - return; - } - } + // `op_failure_scope` (an output parameter) is not provided (set to nullptr) + // since a failure must be propagated regardless of its value. + const Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key_, MergeHelper::kNoBaseValue, + merge_context_->GetOperands(), logger_, statistics_, clock_, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + pinnable_val_ ? pinnable_val_->GetSelf() : nullptr, columns_); + PostprocessMerge(s); +} - { - // `op_failure_scope` (an output parameter) is not provided (set to - // nullptr) since a failure must be propagated regardless of its value. - const Status s = MergeHelper::TimedFullMerge( - merge_operator_, user_key_, &value_of_default, - merge_context_->GetOperands(), pinnable_val_->GetSelf(), logger_, - statistics_, clock_, /* result_operand */ nullptr, - /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - if (!s.ok()) { - if (s.subcode() == Status::SubCode::kMergeOperatorFailed) { - state_ = kMergeOperatorFailed; - } else { - state_ = kCorrupt; - } - return; - } - } +void GetContext::MergeWithPlainBaseValue(const Slice& value) { + assert(do_merge_); + assert(pinnable_val_ || columns_); + assert(!pinnable_val_ || !columns_); - pinnable_val_->PinSelf(); - return; - } + // `op_failure_scope` (an output parameter) is not provided (set to nullptr) + // since a failure must be propagated regardless of its value. + const Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key_, MergeHelper::kPlainBaseValue, value, + merge_context_->GetOperands(), logger_, statistics_, clock_, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + pinnable_val_ ? pinnable_val_->GetSelf() : nullptr, columns_); + PostprocessMerge(s); +} - std::string result; - - { - // `op_failure_scope` (an output parameter) is not provided (set to nullptr) - // since a failure must be propagated regardless of its value. - const Status s = MergeHelper::TimedFullMergeWithEntity( - merge_operator_, user_key_, entity, merge_context_->GetOperands(), - &result, logger_, statistics_, clock_, /* update_num_ops_stats */ true, - /* op_failure_scope */ nullptr); - if (!s.ok()) { - if (s.subcode() == Status::SubCode::kMergeOperatorFailed) { - state_ = kMergeOperatorFailed; - } else { - state_ = kCorrupt; - } - return; - } - } +void GetContext::MergeWithWideColumnBaseValue(const Slice& entity) { + assert(do_merge_); + assert(pinnable_val_ || columns_); + assert(!pinnable_val_ || !columns_); - { - assert(columns_); - const Status s = columns_->SetWideColumnValue(std::move(result)); - if (!s.ok()) { - state_ = kCorrupt; - return; - } - } + // `op_failure_scope` (an output parameter) is not provided (set to nullptr) + // since a failure must be propagated regardless of its value. + const Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key_, MergeHelper::kWideBaseValue, entity, + merge_context_->GetOperands(), logger_, statistics_, clock_, + /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, + pinnable_val_ ? pinnable_val_->GetSelf() : nullptr, columns_); + PostprocessMerge(s); } bool GetContext::GetBlobValue(const Slice& user_key, const Slice& blob_index, @@ -594,22 +562,39 @@ void GetContext::push_operand(const Slice& value, Cleanable* value_pinner) { } void replayGetContextLog(const Slice& replay_log, const Slice& user_key, - GetContext* get_context, Cleanable* value_pinner) { + GetContext* get_context, Cleanable* value_pinner, + SequenceNumber seq_no) { Slice s = replay_log; + Slice ts; + size_t ts_sz = get_context->TimestampSize(); + bool ret = false; + while (s.size()) { auto type = static_cast(*s.data()); s.remove_prefix(1); Slice value; - bool ret = GetLengthPrefixedSlice(&s, &value); + ret = GetLengthPrefixedSlice(&s, &value); assert(ret); - (void)ret; bool dont_care __attribute__((__unused__)); - // Since SequenceNumber is not stored and unknown, we will use - // kMaxSequenceNumber. - get_context->SaveValue( - ParsedInternalKey(user_key, kMaxSequenceNumber, type), value, - &dont_care, value_pinner); + + // Use a copy to prevent modifying user_key. Modification of user_key + // could result to potential cache miss. + std::string user_key_str = user_key.ToString(); + ParsedInternalKey ikey = ParsedInternalKey(user_key_str, seq_no, type); + + // If ts enabled for current cf, there will always be ts appended after each + // piece of value. + if (ts_sz > 0) { + ret = GetLengthPrefixedSlice(&s, &ts); + assert(ts_sz == ts.size()); + assert(ret); + ikey.SetTimestamp(ts); + } + + (void)ret; + + get_context->SaveValue(ikey, value, &dont_care, value_pinner); } } diff --git a/table/get_context.h b/table/get_context.h index 528cd14fd..da41631fc 100644 --- a/table/get_context.h +++ b/table/get_context.h @@ -149,6 +149,8 @@ class GetContext { bool NeedTimestamp() { return timestamp_ != nullptr; } + inline size_t TimestampSize() { return ucmp_->timestamp_size(); } + void SetTimestampFromRangeTombstone(const Slice& timestamp) { assert(timestamp_); timestamp_->assign(timestamp.data(), timestamp.size()); @@ -191,11 +193,21 @@ class GetContext { void push_operand(const Slice& value, Cleanable* value_pinner); private: - void Merge(const Slice* value); - void MergeWithEntity(Slice entity); + // Helper method that postprocesses the results of merge operations, e.g. it + // sets the state correctly upon merge errors. + void PostprocessMerge(const Status& merge_status); + + // The following methods perform the actual merge operation for the + // no base value/plain base value/wide-column base value cases. + void MergeWithNoBaseValue(); + void MergeWithPlainBaseValue(const Slice& value); + void MergeWithWideColumnBaseValue(const Slice& entity); + bool GetBlobValue(const Slice& user_key, const Slice& blob_index, PinnableSlice* blob_value); + void appendToReplayLog(ValueType type, Slice value, Slice ts); + const Comparator* ucmp_; const MergeOperator* merge_operator_; // the merge operations encountered; @@ -240,6 +252,7 @@ class GetContext { // must have been set by calling GetContext::SetReplayLog(). void replayGetContextLog(const Slice& replay_log, const Slice& user_key, GetContext* get_context, - Cleanable* value_pinner = nullptr); + Cleanable* value_pinner = nullptr, + SequenceNumber seq_no = kMaxSequenceNumber); } // namespace ROCKSDB_NAMESPACE diff --git a/table/internal_iterator.h b/table/internal_iterator.h index 8015ed635..060306003 100644 --- a/table/internal_iterator.h +++ b/table/internal_iterator.h @@ -43,6 +43,17 @@ class InternalIteratorBase : public Cleanable { virtual ~InternalIteratorBase() {} + // This iterator will only process range tombstones with sequence + // number <= `read_seqno`. + // Noop for most child classes. + // For range tombstone iterators (TruncatedRangeDelIterator, + // FragmentedRangeTombstoneIterator), will only return range tombstones with + // sequence number <= `read_seqno`. For LevelIterator, it may open new table + // files and create new range tombstone iterators during scanning. It will use + // `read_seqno` as the sequence number for creating new range tombstone + // iterators. + virtual void SetRangeDelReadSeqno(SequenceNumber /* read_seqno */) {} + // An iterator is either positioned at a key/value pair, or // not valid. This method returns true iff the iterator is valid. // Always returns false if !status().ok(). diff --git a/table/iterator_wrapper.h b/table/iterator_wrapper.h index 17abef4ac..a9de3dff3 100644 --- a/table/iterator_wrapper.h +++ b/table/iterator_wrapper.h @@ -30,6 +30,11 @@ class IteratorWrapperBase { } ~IteratorWrapperBase() {} InternalIteratorBase* iter() const { return iter_; } + void SetRangeDelReadSeqno(SequenceNumber read_seqno) { + if (iter_) { + iter_->SetRangeDelReadSeqno(read_seqno); + } + } // Set the underlying Iterator to _iter and return // previous underlying Iterator. @@ -47,6 +52,17 @@ class IteratorWrapperBase { void DeleteIter(bool is_arena_mode) { if (iter_) { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + if (!status_checked_after_invalid_) { + // If this assertion fails, it is likely that you did not check + // iterator status after Valid() returns false. + fprintf(stderr, + "Failed to check status after Valid() returned false from this " + "iterator.\n"); + port::PrintStack(); + std::abort(); + } +#endif if (!is_arena_mode) { delete iter_; } else { @@ -56,7 +72,12 @@ class IteratorWrapperBase { } // Iterator interface methods - bool Valid() const { return valid_; } + bool Valid() const { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + status_checked_after_invalid_ = valid_; +#endif + return valid_; + } Slice key() const { assert(Valid()); return result_.key; @@ -67,6 +88,9 @@ class IteratorWrapperBase { } // Methods below require iter() != nullptr Status status() const { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + status_checked_after_invalid_ = true; +#endif assert(iter_); return iter_->status(); } @@ -178,6 +202,10 @@ class IteratorWrapperBase { InternalIteratorBase* iter_; IterateResult result_; bool valid_; + +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + mutable bool status_checked_after_invalid_ = true; +#endif }; using IteratorWrapper = IteratorWrapperBase; diff --git a/table/merger_test.cc b/table/merger_test.cc index 71dc798e5..29e433c28 100644 --- a/table/merger_test.cc +++ b/table/merger_test.cc @@ -107,7 +107,7 @@ class MergerTest : public testing::Test { } merging_iterator_.reset( - NewMergingIterator(&icomp_, &small_iterators[0], + NewMergingIterator(&icomp_, small_iterators.data(), static_cast(small_iterators.size()))); single_iterator_.reset(new VectorIterator(all_keys_, all_keys_, &icomp_)); } diff --git a/table/merging_iterator.cc b/table/merging_iterator.cc index 505cd76d3..247564fe7 100644 --- a/table/merging_iterator.cc +++ b/table/merging_iterator.cc @@ -135,6 +135,18 @@ class MergingIterator : public InternalIterator { status_.PermitUncheckedError(); } + void SetRangeDelReadSeqno(SequenceNumber read_seqno) override { + for (auto& child : children_) { + // This should only be needed for LevelIterator (iterators from L1+). + child.iter.SetRangeDelReadSeqno(read_seqno); + } + for (auto& child : range_tombstone_iters_) { + if (child) { + child->SetRangeDelReadSeqno(read_seqno); + } + } + } + bool Valid() const override { return current_ != nullptr && status_.ok(); } Status status() const override { return status_; } diff --git a/table/meta_blocks.cc b/table/meta_blocks.cc index cf756cfff..2cbaacec0 100644 --- a/table/meta_blocks.cc +++ b/table/meta_blocks.cc @@ -27,6 +27,8 @@ namespace ROCKSDB_NAMESPACE { const std::string kPropertiesBlockName = "rocksdb.properties"; +// NB: only used with format_version >= 6 +const std::string kIndexBlockName = "rocksdb.index"; // Old property block name for backward compatibility const std::string kPropertiesBlockOldName = "rocksdb.stats"; const std::string kCompressionDictBlockName = "rocksdb.compression_dict"; @@ -211,21 +213,25 @@ void NotifyCollectTableCollectorsOnBlockAdd( bool NotifyCollectTableCollectorsOnFinish( const std::vector>& collectors, - Logger* info_log, PropertyBlockBuilder* builder) { + Logger* info_log, PropertyBlockBuilder* builder, + UserCollectedProperties& user_collected_properties, + UserCollectedProperties& readable_properties) { bool all_succeeded = true; for (auto& collector : collectors) { - UserCollectedProperties user_collected_properties; Status s = collector->Finish(&user_collected_properties); - - all_succeeded = all_succeeded && s.ok(); - if (!s.ok()) { + if (s.ok()) { + for (const auto& prop : collector->GetReadableProperties()) { + readable_properties.insert(prop); + } + builder->Add(user_collected_properties); + } else { LogPropertiesCollectionError(info_log, "Finish" /* method */, collector->Name()); - } else { - builder->Add(user_collected_properties); + if (all_succeeded) { + all_succeeded = false; + } } } - return all_succeeded; } @@ -395,8 +401,8 @@ Status ReadTablePropertiesHelper( // Modified version of BlockFetcher checksum verification // (See write_global_seqno comment above) if (s.ok() && footer.GetBlockTrailerSize() > 0) { - s = VerifyBlockChecksum(footer.checksum_type(), properties_block.data(), - block_size, file->file_name(), handle.offset()); + s = VerifyBlockChecksum(footer, properties_block.data(), block_size, + file->file_name(), handle.offset()); if (s.IsCorruption()) { if (new_table_properties->external_sst_file_global_seqno_offset != 0) { std::string tmp_buf(properties_block.data(), @@ -405,8 +411,8 @@ Status ReadTablePropertiesHelper( new_table_properties->external_sst_file_global_seqno_offset - handle.offset(); EncodeFixed64(&tmp_buf[static_cast(global_seqno_offset)], 0); - s = VerifyBlockChecksum(footer.checksum_type(), tmp_buf.data(), - block_size, file->file_name(), handle.offset()); + s = VerifyBlockChecksum(footer, tmp_buf.data(), block_size, + file->file_name(), handle.offset()); } } } diff --git a/table/meta_blocks.h b/table/meta_blocks.h index 962a31638..0a404dc9c 100644 --- a/table/meta_blocks.h +++ b/table/meta_blocks.h @@ -32,6 +32,7 @@ struct TableProperties; // Meta block names for metaindex extern const std::string kPropertiesBlockName; +extern const std::string kIndexBlockName; extern const std::string kPropertiesBlockOldName; extern const std::string kCompressionDictBlockName; extern const std::string kRangeDelBlockName; @@ -97,9 +98,13 @@ void NotifyCollectTableCollectorsOnBlockAdd( // NotifyCollectTableCollectorsOnFinish() triggers the `Finish` event for all // property collectors. The collected properties will be added to `builder`. +// It will also populate `user_collected_properties` and `readable_properties` +// with the collected properties. bool NotifyCollectTableCollectorsOnFinish( const std::vector>& collectors, - Logger* info_log, PropertyBlockBuilder* builder); + Logger* info_log, PropertyBlockBuilder* builder, + UserCollectedProperties& user_collected_properties, + UserCollectedProperties& readable_properties); // Read table properties from a file using known BlockHandle. // @returns a status to indicate if the operation succeeded. On success, diff --git a/table/mock_table.cc b/table/mock_table.cc index c251ea108..1971c00fc 100644 --- a/table/mock_table.cc +++ b/table/mock_table.cc @@ -59,7 +59,7 @@ class MockTableReader : public TableReader { std::shared_ptr GetTableProperties() const override; - ~MockTableReader() {} + ~MockTableReader() = default; private: const KVVector& table_; @@ -134,7 +134,7 @@ class MockTableBuilder : public TableBuilder { } // REQUIRES: Either Finish() or Abandon() has been called. - ~MockTableBuilder() {} + ~MockTableBuilder() = default; // Add key,value to the table being constructed. // REQUIRES: key is after any previously added key according to comparator. @@ -230,7 +230,13 @@ Status MockTableReader::Get(const ReadOptions&, const Slice& key, std::shared_ptr MockTableReader::GetTableProperties() const { - return std::shared_ptr(new TableProperties()); + TableProperties* tp = new TableProperties(); + tp->num_entries = table_.size(); + tp->num_range_deletions = 0; + tp->raw_key_size = 1; + tp->raw_value_size = 1; + + return std::shared_ptr(tp); } MockTableFactory::MockTableFactory() @@ -299,8 +305,7 @@ Status MockTableFactory::GetIDFromFile(RandomAccessFileReader* file, uint32_t* id) const { char buf[4]; Slice result; - Status s = file->Read(IOOptions(), 0, 4, &result, buf, nullptr, - Env::IO_TOTAL /* rate_limiter_priority */); + Status s = file->Read(IOOptions(), 0, 4, &result, buf, nullptr); assert(result.size() == 4); *id = DecodeFixed32(buf); return s; diff --git a/table/plain/plain_table_builder.cc b/table/plain/plain_table_builder.cc index 126098a86..1e61773d6 100644 --- a/table/plain/plain_table_builder.cc +++ b/table/plain/plain_table_builder.cc @@ -5,8 +5,7 @@ #include "table/plain/plain_table_builder.h" -#include - +#include #include #include #include @@ -119,9 +118,12 @@ PlainTableBuilder::PlainTableBuilder( for (auto& factory : *int_tbl_prop_collector_factories) { assert(factory); - table_properties_collectors_.emplace_back( + std::unique_ptr collector{ factory->CreateIntTblPropCollector(column_family_id, - level_at_creation)); + level_at_creation)}; + if (collector) { + table_properties_collectors_.emplace_back(std::move(collector)); + } } } @@ -265,19 +267,24 @@ Status PlainTableBuilder::Finish() { PropertyBlockBuilder property_block_builder; // -- Add basic properties property_block_builder.AddTableProperty(properties_); - + // -- Add eixsting user collected properties property_block_builder.Add(properties_.user_collected_properties); - - // -- Add user collected properties + // -- Add more user collected properties + UserCollectedProperties more_user_collected_properties; NotifyCollectTableCollectorsOnFinish( - table_properties_collectors_, ioptions_.logger, &property_block_builder); + table_properties_collectors_, ioptions_.logger, &property_block_builder, + more_user_collected_properties, properties_.readable_properties); + properties_.user_collected_properties.insert( + more_user_collected_properties.begin(), + more_user_collected_properties.end()); // -- Write property block BlockHandle property_block_handle; - IOStatus s = WriteBlock(property_block_builder.Finish(), file_, &offset_, + io_status_ = WriteBlock(property_block_builder.Finish(), file_, &offset_, &property_block_handle); - if (!s.ok()) { - return static_cast(s); + if (!io_status_.ok()) { + status_ = io_status_; + return status_; } meta_index_builer.Add(kPropertiesBlockName, property_block_handle); @@ -293,8 +300,12 @@ Status PlainTableBuilder::Finish() { // Write Footer // no need to write out new footer if we're using default checksum FooterBuilder footer; - footer.Build(kPlainTableMagicNumber, /* format_version */ 0, offset_, - kNoChecksum, metaindex_block_handle); + Status s = footer.Build(kPlainTableMagicNumber, /* format_version */ 0, + offset_, kNoChecksum, metaindex_block_handle); + if (!s.ok()) { + status_ = s; + return status_; + } io_status_ = file_->Append(footer.GetSlice()); if (io_status_.ok()) { offset_ += footer.GetSlice().size(); diff --git a/table/plain/plain_table_builder.h b/table/plain/plain_table_builder.h index 27c07898f..fb7ea63be 100644 --- a/table/plain/plain_table_builder.h +++ b/table/plain/plain_table_builder.h @@ -85,16 +85,7 @@ class PlainTableBuilder : public TableBuilder { // Finish() call, returns the size of the final generated file. uint64_t FileSize() const override; - TableProperties GetTableProperties() const override { - TableProperties ret = properties_; - for (const auto& collector : table_properties_collectors_) { - for (const auto& prop : collector->GetReadableProperties()) { - ret.readable_properties.insert(prop); - } - collector->Finish(&ret.user_collected_properties).PermitUncheckedError(); - } - return ret; - } + TableProperties GetTableProperties() const override { return properties_; } bool SaveIndexInFile() const { return store_index_in_file_; } @@ -158,4 +149,3 @@ class PlainTableBuilder : public TableBuilder { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index 80aa9cb8e..730aec028 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -5,8 +5,7 @@ #include "table/plain/plain_table_factory.h" -#include - +#include #include #include "db/dbformat.h" @@ -157,7 +156,7 @@ static int RegisterBuiltinMemTableRepFactory(ObjectLibrary& library, AsPattern(VectorRepFactory::kClassName(), VectorRepFactory::kNickName()), [](const std::string& uri, std::unique_ptr* guard, std::string* /*errmsg*/) { - auto colon = uri.find(":"); + auto colon = uri.find(':'); if (colon != std::string::npos) { size_t count = ParseSizeT(uri.substr(colon + 1)); guard->reset(new VectorRepFactory(count)); @@ -170,7 +169,7 @@ static int RegisterBuiltinMemTableRepFactory(ObjectLibrary& library, AsPattern(SkipListFactory::kClassName(), SkipListFactory::kNickName()), [](const std::string& uri, std::unique_ptr* guard, std::string* /*errmsg*/) { - auto colon = uri.find(":"); + auto colon = uri.find(':'); if (colon != std::string::npos) { size_t lookahead = ParseSizeT(uri.substr(colon + 1)); guard->reset(new SkipListFactory(lookahead)); @@ -184,7 +183,7 @@ static int RegisterBuiltinMemTableRepFactory(ObjectLibrary& library, [](const std::string& uri, std::unique_ptr* guard, std::string* /*errmsg*/) { // Expecting format: hash_linkedlist: - auto colon = uri.find(":"); + auto colon = uri.find(':'); if (colon != std::string::npos) { size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1)); guard->reset(NewHashLinkListRepFactory(hash_bucket_count)); @@ -198,7 +197,7 @@ static int RegisterBuiltinMemTableRepFactory(ObjectLibrary& library, [](const std::string& uri, std::unique_ptr* guard, std::string* /*errmsg*/) { // Expecting format: prefix_hash: - auto colon = uri.find(":"); + auto colon = uri.find(':'); if (colon != std::string::npos) { size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1)); guard->reset(NewHashSkipListRepFactory(hash_bucket_count)); diff --git a/table/plain/plain_table_key_coding.cc b/table/plain/plain_table_key_coding.cc index a40968a60..0ac423191 100644 --- a/table/plain/plain_table_key_coding.cc +++ b/table/plain/plain_table_key_coding.cc @@ -215,8 +215,7 @@ bool PlainTableFileReader::ReadNonMmap(uint32_t file_offset, uint32_t len, // TODO: rate limit plain table reads. Status s = file_info_->file->Read(IOOptions(), file_offset, size_to_read, - &read_result, new_buffer->buf.get(), nullptr, - Env::IO_TOTAL /* rate_limiter_priority */); + &read_result, new_buffer->buf.get(), nullptr); if (!s.ok()) { status_ = s; return false; diff --git a/table/plain/plain_table_reader.cc b/table/plain/plain_table_reader.cc index 2f0379f72..b917fce34 100644 --- a/table/plain/plain_table_reader.cc +++ b/table/plain/plain_table_reader.cc @@ -284,9 +284,9 @@ void PlainTableReader::FillBloom(const std::vector& prefix_hashes) { Status PlainTableReader::MmapDataIfNeeded() { if (file_info_.is_mmap_mode) { // Get mmapped memory. - return file_info_.file->Read( - IOOptions(), 0, static_cast(file_size_), &file_info_.file_data, - nullptr, nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + return file_info_.file->Read(IOOptions(), 0, + static_cast(file_size_), + &file_info_.file_data, nullptr, nullptr); } return Status::OK(); } @@ -454,7 +454,9 @@ Status PlainTableReader::GetOffset(PlainTableKeyDecoder* decoder, ParsedInternalKey parsed_target; Status s = ParseInternalKey(target, &parsed_target, false /* log_err_key */); // TODO - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } // The key is between [low, high). Do a binary search between it. while (high - low > 1) { @@ -591,7 +593,9 @@ Status PlainTableReader::Get(const ReadOptions& /*ro*/, const Slice& target, ParsedInternalKey parsed_target; s = ParseInternalKey(target, &parsed_target, false /* log_err_key */); // TODO - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } Slice found_value; while (offset < file_info_.data_end_offset) { @@ -642,7 +646,7 @@ PlainTableIterator::PlainTableIterator(PlainTableReader* table, next_offset_ = offset_ = table_->file_info_.data_end_offset; } -PlainTableIterator::~PlainTableIterator() {} +PlainTableIterator::~PlainTableIterator() = default; bool PlainTableIterator::Valid() const { return offset_ < table_->file_info_.data_end_offset && diff --git a/table/sst_file_dumper.cc b/table/sst_file_dumper.cc index 85a264d59..821fff5b3 100644 --- a/table/sst_file_dumper.cc +++ b/table/sst_file_dumper.cc @@ -16,6 +16,8 @@ #include "db/blob/blob_index.h" #include "db/memtable.h" +#include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "db/write_batch_internal.h" #include "options/cf_options.h" #include "port/port.h" @@ -36,6 +38,7 @@ #include "table/table_reader.h" #include "util/compression.h" #include "util/random.h" +#include "util/udt_util.h" namespace ROCKSDB_NAMESPACE { @@ -68,6 +71,7 @@ extern const uint64_t kBlockBasedTableMagicNumber; extern const uint64_t kLegacyBlockBasedTableMagicNumber; extern const uint64_t kPlainTableMagicNumber; extern const uint64_t kLegacyPlainTableMagicNumber; +extern const uint64_t kCuckooTableMagicNumber; const char* testFileName = "test_file_name"; @@ -109,8 +113,7 @@ Status SstFileDumper::GetTableReader(const std::string& file_path) { uint64_t prefetch_off = file_size - prefetch_size; IOOptions opts; s = prefetch_buffer.Prefetch(opts, file_.get(), prefetch_off, - static_cast(prefetch_size), - Env::IO_TOTAL /* rate_limiter_priority */); + static_cast(prefetch_size)); s = ReadFooterFromFile(opts, file_.get(), *fs, &prefetch_buffer, file_size, &footer); @@ -121,9 +124,15 @@ Status SstFileDumper::GetTableReader(const std::string& file_path) { if (s.ok()) { if (magic_number == kPlainTableMagicNumber || - magic_number == kLegacyPlainTableMagicNumber) { + magic_number == kLegacyPlainTableMagicNumber || + magic_number == kCuckooTableMagicNumber) { soptions_.use_mmap_reads = true; + if (magic_number == kCuckooTableMagicNumber) { + fopts = soptions_; + fopts.temperature = file_temp_; + } + fs->NewRandomAccessFile(file_path, fopts, &file, nullptr); file_.reset(new RandomAccessFileReader(std::move(file), file_path)); } @@ -424,6 +433,13 @@ Status SstFileDumper::SetTableOptionsByMagicNumber( if (!silent_) { fprintf(stdout, "Sst file format: plain table\n"); } + } else if (table_magic_number == kCuckooTableMagicNumber) { + ioptions_.allow_mmap_reads = true; + + options_.table_factory.reset(NewCuckooTableFactory()); + if (!silent_) { + fprintf(stdout, "Sst file format: cuckoo table\n"); + } } else { char error_msg_buffer[80]; snprintf(error_msg_buffer, sizeof(error_msg_buffer) - 1, @@ -457,10 +473,20 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, read_options_, moptions_.prefix_extractor.get(), /*arena=*/nullptr, /*skip_filters=*/false, TableReaderCaller::kSSTDumpTool); + + const Comparator* ucmp = internal_comparator_.user_comparator(); + size_t ts_sz = ucmp->timestamp_size(); + + Slice from_slice = from_key; + Slice to_slice = to_key; + std::string from_key_buf, to_key_buf; + auto [from, to] = MaybeAddTimestampsToRange( + has_from ? &from_slice : nullptr, has_to ? &to_slice : nullptr, ts_sz, + &from_key_buf, &to_key_buf); uint64_t i = 0; - if (has_from) { + if (from.has_value()) { InternalKey ikey; - ikey.SetMinPossibleForUserKey(from_key); + ikey.SetMinPossibleForUserKey(from.value()); iter->Seek(ikey.Encode()); } else { iter->SeekToFirst(); @@ -469,7 +495,9 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, Slice key = iter->key(); Slice value = iter->value(); ++i; - if (read_num > 0 && i > read_num) break; + if (read_num > 0 && i > read_num) { + break; + } ParsedInternalKey ikey; Status pik_status = ParseInternalKey(key, &ikey, true /* log_err_key */); @@ -484,15 +512,29 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, } // If end marker was specified, we stop before it - if (has_to && BytewiseComparator()->Compare(ikey.user_key, to_key) >= 0) { + if (to.has_value() && ucmp->Compare(ikey.user_key, to.value()) >= 0) { break; } if (print_kv) { if (!decode_blob_index_ || ikey.type != kTypeBlobIndex) { - fprintf(stdout, "%s => %s\n", - ikey.DebugString(true, output_hex_).c_str(), - value.ToString(output_hex_).c_str()); + if (ikey.type == kTypeWideColumnEntity) { + std::ostringstream oss; + const Status s = WideColumnsHelper::DumpSliceAsWideColumns( + iter->value(), oss, output_hex_); + if (!s.ok()) { + fprintf(stderr, "%s => error deserializing wide columns\n", + ikey.DebugString(true, output_hex_).c_str()); + continue; + } + fprintf(stdout, "%s => %s\n", + ikey.DebugString(true, output_hex_).c_str(), + oss.str().c_str()); + } else { + fprintf(stdout, "%s => %s\n", + ikey.DebugString(true, output_hex_).c_str(), + value.ToString(output_hex_).c_str()); + } } else { BlobIndex blob_index; diff --git a/table/sst_file_dumper.h b/table/sst_file_dumper.h index 1e78959d1..bd97d817d 100644 --- a/table/sst_file_dumper.h +++ b/table/sst_file_dumper.h @@ -88,7 +88,7 @@ class SstFileDumper { std::unique_ptr table_reader_; std::unique_ptr file_; - const ImmutableOptions ioptions_; + ImmutableOptions ioptions_; const MutableCFOptions moptions_; ReadOptions read_options_; InternalKeyComparator internal_comparator_; diff --git a/table/sst_file_reader.cc b/table/sst_file_reader.cc index 533b7cd6a..d23c58deb 100644 --- a/table/sst_file_reader.cc +++ b/table/sst_file_reader.cc @@ -36,7 +36,7 @@ struct SstFileReader::Rep { SstFileReader::SstFileReader(const Options& options) : rep_(new Rep(options)) {} -SstFileReader::~SstFileReader() {} +SstFileReader::~SstFileReader() = default; Status SstFileReader::Open(const std::string& file_path) { auto r = rep_.get(); diff --git a/table/sst_file_reader_test.cc b/table/sst_file_reader_test.cc index ba81d7815..36a7975cf 100644 --- a/table/sst_file_reader_test.cc +++ b/table/sst_file_reader_test.cc @@ -304,6 +304,7 @@ class SstFileReaderTimestampTest : public testing::Test { } ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); } protected: diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index d187b741e..1ef0f98aa 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -9,6 +9,8 @@ #include "db/db_impl/db_impl.h" #include "db/dbformat.h" +#include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "file/writable_file_writer.h" #include "rocksdb/file_system.h" #include "rocksdb/table.h" @@ -81,7 +83,8 @@ struct SstFileWriter::Rep { assert(value_type == kTypeValue || value_type == kTypeMerge || value_type == kTypeDeletion || - value_type == kTypeDeletionWithTimestamp); + value_type == kTypeDeletionWithTimestamp || + value_type == kTypeWideColumnEntity); constexpr SequenceNumber sequence_number = 0; @@ -130,6 +133,21 @@ struct SstFileWriter::Rep { return AddImpl(user_key_with_ts, value, value_type); } + Status AddEntity(const Slice& user_key, const WideColumns& columns) { + WideColumns sorted_columns(columns); + WideColumnsHelper::SortColumns(sorted_columns); + + std::string entity; + const Status s = WideColumnSerialization::Serialize(sorted_columns, entity); + if (!s.ok()) { + return s; + } + if (entity.size() > size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("wide column entity is too large"); + } + return Add(user_key, entity, kTypeWideColumnEntity); + } + Status DeleteRangeImpl(const Slice& begin_key, const Slice& end_key) { if (!builder) { return Status::InvalidArgument("File is not opened"); @@ -371,6 +389,11 @@ Status SstFileWriter::Put(const Slice& user_key, const Slice& timestamp, return rep_->Add(user_key, timestamp, value, ValueType::kTypeValue); } +Status SstFileWriter::PutEntity(const Slice& user_key, + const WideColumns& columns) { + return rep_->AddEntity(user_key, columns); +} + Status SstFileWriter::Merge(const Slice& user_key, const Slice& value) { return rep_->Add(user_key, value, ValueType::kTypeMerge); } diff --git a/table/table_properties.cc b/table/table_properties.cc index 06ea13f7c..17a13543d 100644 --- a/table/table_properties.cc +++ b/table/table_properties.cc @@ -109,6 +109,10 @@ std::string TableProperties::ToString(const std::string& prop_delim, AppendProperty(result, "comparator name", comparator_name.empty() ? std::string("N/A") : comparator_name, prop_delim, kv_delim); + AppendProperty(result, "user defined timestamps persisted", + user_defined_timestamps_persisted ? std::string("true") + : std::string("false"), + prop_delim, kv_delim); AppendProperty( result, "merge operator name", diff --git a/table/table_reader.h b/table/table_reader.h index 53c522052..87610f4fe 100644 --- a/table/table_reader.h +++ b/table/table_reader.h @@ -60,11 +60,17 @@ class TableReader { size_t compaction_readahead_size = 0, bool allow_unprepared_value = false) = 0; + // read_options.snapshot needs to outlive this call. virtual FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator( const ReadOptions& /*read_options*/) { return nullptr; } + virtual FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator( + SequenceNumber /* read_seqno */, const Slice* /* timestamp */) { + return nullptr; + } + // Given a key, return an approximate byte offset in the file where // the data for that key begins (or would begin if the key were // present in the file). The returned value is in terms of file diff --git a/table/table_test.cc b/table/table_test.cc index 9610ba767..298e25fbd 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -10,10 +10,10 @@ #include "rocksdb/table.h" #include -#include -#include #include +#include +#include #include #include #include @@ -52,6 +52,7 @@ #include "table/block_based/block.h" #include "table/block_based/block_based_table_builder.h" #include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_based_table_iterator.h" #include "table/block_based/block_based_table_reader.h" #include "table/block_based/block_builder.h" #include "table/block_based/filter_policy_internal.h" @@ -186,7 +187,7 @@ class Constructor { public: explicit Constructor(const Comparator* cmp) : data_(stl_wrappers::LessOfComparator(cmp)) {} - virtual ~Constructor() {} + virtual ~Constructor() = default; void Add(const std::string& key, const Slice& value) { data_[key] = value.ToString(); @@ -295,8 +296,8 @@ class KeyConvertingIterator : public InternalIterator { bool arena_mode_; // No copying allowed - KeyConvertingIterator(const KeyConvertingIterator&); - void operator=(const KeyConvertingIterator&); + KeyConvertingIterator(const KeyConvertingIterator&) = delete; + void operator=(const KeyConvertingIterator&) = delete; }; // `BlockConstructor` APIs always accept/return user keys. @@ -345,7 +346,7 @@ class BlockConstructor : public Constructor { std::string data_; Block* block_; - BlockConstructor(); + BlockConstructor() = delete; }; class TableConstructor : public Constructor { @@ -469,6 +470,7 @@ class TableConstructor : public Constructor { } BlockCacheTracer block_cache_tracer_; + Env* env_; private: void Reset() { @@ -487,11 +489,10 @@ class TableConstructor : public Constructor { bool convert_to_internal_key_; int level_; - TableConstructor(); + TableConstructor() = delete; static uint64_t cur_file_num_; EnvOptions soptions; - Env* env_; }; uint64_t TableConstructor::cur_file_num_ = 1; @@ -930,13 +931,17 @@ class HarnessTest : public testing::Test { InternalIterator* iter = constructor_->NewIterator(); ASSERT_TRUE(!iter->Valid()); stl_wrappers::KVMap::const_iterator model_iter = data.begin(); - if (kVerbose) fprintf(stderr, "---\n"); + if (kVerbose) { + fprintf(stderr, "---\n"); + } for (int i = 0; i < 200; i++) { const int toss = rnd->Uniform(support_prev_ ? 5 : 3); switch (toss) { case 0: { if (iter->Valid()) { - if (kVerbose) fprintf(stderr, "Next\n"); + if (kVerbose) { + fprintf(stderr, "Next\n"); + } iter->Next(); ASSERT_OK(iter->status()); ++model_iter; @@ -946,7 +951,9 @@ class HarnessTest : public testing::Test { } case 1: { - if (kVerbose) fprintf(stderr, "SeekToFirst\n"); + if (kVerbose) { + fprintf(stderr, "SeekToFirst\n"); + } iter->SeekToFirst(); ASSERT_OK(iter->status()); model_iter = data.begin(); @@ -957,8 +964,9 @@ class HarnessTest : public testing::Test { case 2: { std::string key = PickRandomKey(rnd, keys); model_iter = data.lower_bound(key); - if (kVerbose) + if (kVerbose) { fprintf(stderr, "Seek '%s'\n", EscapeString(key).c_str()); + } iter->Seek(Slice(key)); ASSERT_OK(iter->status()); ASSERT_EQ(ToString(data, model_iter), ToString(iter)); @@ -967,7 +975,9 @@ class HarnessTest : public testing::Test { case 3: { if (iter->Valid()) { - if (kVerbose) fprintf(stderr, "Prev\n"); + if (kVerbose) { + fprintf(stderr, "Prev\n"); + } iter->Prev(); ASSERT_OK(iter->status()); if (model_iter == data.begin()) { @@ -981,7 +991,9 @@ class HarnessTest : public testing::Test { } case 4: { - if (kVerbose) fprintf(stderr, "SeekToLast\n"); + if (kVerbose) { + fprintf(stderr, "SeekToLast\n"); + } iter->SeekToLast(); ASSERT_OK(iter->status()); if (keys.empty()) { @@ -1125,9 +1137,7 @@ class BlockBasedTableTest : public BlockBasedTableTestBase, virtual public ::testing::WithParamInterface { public: - BlockBasedTableTest() : format_(GetParam()) { - env_ = ROCKSDB_NAMESPACE::Env::Default(); - } + BlockBasedTableTest() : format_(GetParam()) { env_ = Env::Default(); } BlockBasedTableOptions GetBlockBasedTableOptions() { BlockBasedTableOptions options; @@ -1253,7 +1263,7 @@ class FileChecksumTestHelper { public: FileChecksumTestHelper(bool convert_to_internal_key = false) : convert_to_internal_key_(convert_to_internal_key) {} - ~FileChecksumTestHelper() {} + ~FileChecksumTestHelper() = default; void CreateWritableFile() { sink_ = new test::StringSink(); @@ -1330,7 +1340,7 @@ class FileChecksumTestHelper { uint64_t offset = 0; Status s; s = file_reader_->Read(IOOptions(), offset, 2048, &result, scratch.get(), - nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + nullptr); if (!s.ok()) { return s; } @@ -1338,8 +1348,7 @@ class FileChecksumTestHelper { file_checksum_generator->Update(scratch.get(), result.size()); offset += static_cast(result.size()); s = file_reader_->Read(IOOptions(), offset, 2048, &result, scratch.get(), - nullptr, - Env::IO_TOTAL /* rate_limiter_priority */); + nullptr); if (!s.ok()) { return s; } @@ -1438,7 +1447,7 @@ TestIds GetUniqueId(TableProperties* tp, std::unordered_set* seen, std::string euid; EXPECT_OK(GetExtendedUniqueIdFromTableProperties(*tp, &euid)); EXPECT_EQ(euid.size(), 24U); - t.external_id[0] = DecodeFixed64(&euid[0]); + t.external_id[0] = DecodeFixed64(euid.data()); t.external_id[1] = DecodeFixed64(&euid[8]); t.external_id[2] = DecodeFixed64(&euid[16]); @@ -1446,7 +1455,7 @@ TestIds GetUniqueId(TableProperties* tp, std::unordered_set* seen, EXPECT_OK(GetUniqueIdFromTableProperties(*tp, &uid)); EXPECT_EQ(uid.size(), 16U); EXPECT_EQ(uid, euid.substr(0, 16)); - EXPECT_EQ(t.external_id[0], DecodeFixed64(&uid[0])); + EXPECT_EQ(t.external_id[0], DecodeFixed64(uid.data())); EXPECT_EQ(t.external_id[1], DecodeFixed64(&uid[8])); } // All these should be effectively random @@ -1931,19 +1940,19 @@ void AssertKeysInCache(BlockBasedTable* table_reader, const std::vector& keys_not_in_cache, bool convert = false) { if (convert) { - for (auto key : keys_in_cache) { + for (const auto& key : keys_in_cache) { InternalKey ikey(key, kMaxSequenceNumber, kTypeValue); ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), ikey.Encode())); } - for (auto key : keys_not_in_cache) { + for (const auto& key : keys_not_in_cache) { InternalKey ikey(key, kMaxSequenceNumber, kTypeValue); ASSERT_TRUE(!table_reader->TEST_KeyInCache(ReadOptions(), ikey.Encode())); } } else { - for (auto key : keys_in_cache) { + for (const auto& key : keys_in_cache) { ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), key)); } - for (auto key : keys_not_in_cache) { + for (const auto& key : keys_not_in_cache) { ASSERT_TRUE(!table_reader->TEST_KeyInCache(ReadOptions(), key)); } } @@ -3141,6 +3150,483 @@ TEST_P(BlockBasedTableTest, TracingGetTest) { c.ResetTableReader(); } +void GenerateKVMap(TableConstructor* c) { + int num_block = 100; + Random rnd(101); + uint32_t key = 0; + for (int block = 0; block < num_block; block++) { + for (int i = 0; i < 16; i++) { + char k[9] = {0}; + // Internal key is constructed directly from this key, + // and internal key size is required to be >= 8 bytes, + // so use %08u as the format string. + snprintf(k, sizeof(k), "%08u", key); + std::string v = rnd.RandomString(256); + InternalKey ikey(std::string(k), 0, kTypeValue); + c->Add(ikey.Encode().ToString(), rnd.RandomString(256)); + key++; + } + } +} + +void WarmUpCache(TableConstructor* c, const MutableCFOptions& moptions, + const std::vector& warm_keys) { + ReadOptions ro; + std::unique_ptr iter(c->GetTableReader()->NewIterator( + ro, moptions.prefix_extractor.get(), nullptr, false, + TableReaderCaller::kUncategorized)); + size_t i = 0; + while (i < warm_keys.size()) { + InternalKey ikey(warm_keys[i], 0, kTypeValue); + iter->Seek(ikey.Encode().ToString()); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + i++; + } +} + +TEST_P(BlockBasedTableTest, BlockCacheLookupSeqScans) { + Options options; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.create_if_missing = true; + options.statistics = CreateDBStatistics(); + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_options.block_cache = NewLRUCache(1024 * 1024, 0); + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, true)); + table_options.block_align = true; + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + + TableConstructor c(BytewiseComparator()); + GenerateKVMap(&c); + + std::vector keys; + stl_wrappers::KVMap kvmap; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + const InternalKeyComparator internal_comparator(options.comparator); + + c.Finish(options, ioptions, moptions, table_options, internal_comparator, + &keys, &kvmap); + + BlockBasedTable* bbt = reinterpret_cast(c.GetTableReader()); + BlockHandle block_handle; + + ReadOptions read_options; + read_options.auto_readahead_size = true; + Slice ub = Slice("00000805"); + Slice* ub_ptr = &ub; + read_options.iterate_upper_bound = ub_ptr; + read_options.readahead_size = 16384; + uint64_t buffer_offset; + size_t buffer_len; + + // Test various functionalities - + // 5 blocks prefetched - Current + 4 additional (readahead_size). + { + // Check the behavior when it's - + // Miss(200), Hit(210), Hit(225), Hit(240), Hit(255). + // It should only prefetch current block (200). + { + std::vector warm_keys{"00000210", "00000225", "00000240", + "00000255"}; + WarmUpCache(&c, moptions, warm_keys); + + ASSERT_OK(options.statistics->Reset()); + + std::unique_ptr iter(c.GetTableReader()->NewIterator( + read_options, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + // Seek key - + InternalKey ikey("00000200", 0, kTypeValue); + auto kv_iter = kvmap.find(ikey.Encode().ToString()); + + iter->Seek(kv_iter->first); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + + FilePrefetchBuffer* prefetch_buffer = + (reinterpret_cast(iter.get())) + ->prefetch_buffer(); + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, block_handle); + // It won't prefetch the data of cache hit. + // One block data. + ASSERT_EQ(buffer_len, 4096); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED), + 1); + } + + { + // Check the behavior when it's - + // First Prefetch - Miss(315), Miss(330), Miss(345), Hit(360), Hit(375), + // Second Prefetch - Miss(390), Miss(405) ... + // First prefetch should only prefetch from 315 to 345. + std::vector warm_keys{"00000360", "00000375"}; + WarmUpCache(&c, moptions, warm_keys); + + std::unique_ptr iter(c.GetTableReader()->NewIterator( + read_options, moptions.prefix_extractor.get(), nullptr, false, + TableReaderCaller::kUncategorized)); + + // Seek key - + InternalKey ikey("00000315", 0, kTypeValue); + auto kv_iter = kvmap.find(ikey.Encode().ToString()); + + iter->Seek(kv_iter->first); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + + FilePrefetchBuffer* prefetch_buffer = + (reinterpret_cast(iter.get())) + ->prefetch_buffer(); + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, block_handle); + + // It won't prefetch the data of cache hit. + // 3 blocks data. + ASSERT_EQ(buffer_len, 12288); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + for (; kv_iter != kvmap.end() && iter->Valid(); kv_iter++) { + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + iter->Next(); + ASSERT_OK(iter->status()); + + if (iter->user_key().ToString() == "00000400") { + break; + } + } + + // Second Prefetch. + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, block_handle); + ASSERT_EQ(buffer_offset, 106496); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED), + 1); + } + } + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, BlockCacheLookupAsyncScansSeek) { + Options options; + TableConstructor c(BytewiseComparator()); + std::unique_ptr env( + new CompositeEnvWrapper(c.env_, FileSystem::Default())); + options.env = env.get(); + options.statistics = CreateDBStatistics(); + c.env_ = env.get(); + + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.create_if_missing = true; + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_options.block_cache = NewLRUCache(1024 * 1024, 0); + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, true)); + table_options.block_align = true; + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + + GenerateKVMap(&c); + + std::vector keys; + stl_wrappers::KVMap kvmap; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + const InternalKeyComparator internal_comparator(options.comparator); + + c.Finish(options, ioptions, moptions, table_options, internal_comparator, + &keys, &kvmap); + + BlockBasedTable* bbt = reinterpret_cast(c.GetTableReader()); + BlockHandle block_handle; + + ReadOptions read_options; + read_options.auto_readahead_size = true; + Slice ub = Slice("00000805"); + Slice* ub_ptr = &ub; + read_options.iterate_upper_bound = ub_ptr; + read_options.readahead_size = 16384; + read_options.async_io = true; + uint64_t buffer_offset; + size_t buffer_len; + + // Test Various functionalities - + // 3 blocks prefetched - Current + 2 additional (readahead_size/2). + { + // Check the behavior when it's - + // 1st Prefetch - Miss(200), Hit(210), Hit(225), + // 2nd Prefetch - Hit(240), Hit(255) + // First Prefetch will be for 200 offset. + // Second prefetch will be 0. + { + std::vector warm_keys{"00000210", "00000225", "00000240", + "00000255"}; + WarmUpCache(&c, moptions, warm_keys); + + ASSERT_OK(options.statistics->Reset()); + + std::unique_ptr iter(c.GetTableReader()->NewIterator( + read_options, moptions.prefix_extractor.get(), nullptr, false, + TableReaderCaller::kUncategorized)); + + // Seek key - + InternalKey ikey("00000200", 0, kTypeValue); + auto kv_iter = kvmap.find(ikey.Encode().ToString()); + + iter->Seek(kv_iter->first); + ASSERT_TRUE(iter->status().IsTryAgain()); + iter->Seek(kv_iter->first); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + + FilePrefetchBuffer* prefetch_buffer = + (reinterpret_cast(iter.get())) + ->prefetch_buffer(); + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, block_handle); + ASSERT_EQ(buffer_len, 4096); + ASSERT_EQ(buffer_offset, block_handle.offset()); + prefetch_buffer->TEST_GetBufferOffsetandSize(1, buffer_offset, + buffer_len); + ASSERT_EQ(buffer_len, 0); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED), + 2); + } + { + // Check the behavior when it's - + // First Prefetch - Miss(315), Miss(330), Hit(345), + // Second Prefetch - Miss(360), Miss(375), ... + // First prefetch should only prefetch from 315 to 330. + // Second prefetch should start from 360. + std::vector warm_keys{"00000345"}; + WarmUpCache(&c, moptions, warm_keys); + + std::unique_ptr iter(c.GetTableReader()->NewIterator( + read_options, moptions.prefix_extractor.get(), nullptr, false, + TableReaderCaller::kUncategorized)); + + // Seek key - + InternalKey ikey("00000315", 0, kTypeValue); + auto kv_iter = kvmap.find(ikey.Encode().ToString()); + + iter->Seek(kv_iter->first); + ASSERT_TRUE(iter->status().IsTryAgain()); + iter->Seek(kv_iter->first); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + + FilePrefetchBuffer* prefetch_buffer = + (reinterpret_cast(iter.get())) + ->prefetch_buffer(); + { + // 1st Buffer Verification. + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, + block_handle); + ASSERT_EQ(buffer_len, 8192); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + // 2nd Buffer Verification. + prefetch_buffer->TEST_GetBufferOffsetandSize(1, buffer_offset, + buffer_len); + InternalKey ikey_tmp("00000360", 0, kTypeValue); + bbt->TEST_GetDataBlockHandle(read_options, ikey_tmp.Encode().ToString(), + block_handle); + ASSERT_EQ(buffer_len, 8192); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED), + 1); + } + } + + { + // Check the behavior when it's - + // First Prefetch - Miss(495), Miss(510), Hit(525), prefetch len- 8192 + // Second Prefetch async - Miss(540), Miss(555), - 8192 + // Third Prefetch Async - Hit(570), Miss(585), - 4096 + // 4th Prefetch Async - Hit(600), Miss(615), - 4096 + // 5th Prefetch Async - Miss(630), Miss(645) - 8192 + std::vector warm_keys{"00000525", "00000570", "00000600"}; + WarmUpCache(&c, moptions, warm_keys); + + std::unique_ptr iter(c.GetTableReader()->NewIterator( + read_options, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + // Seek key - + InternalKey ikey("00000495", 0, kTypeValue); + auto kv_iter = kvmap.find(ikey.Encode().ToString()); + + // First and Second Prefetch. + iter->Seek(kv_iter->first); + ASSERT_TRUE(iter->status().IsTryAgain()); + iter->Seek(kv_iter->first); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + + FilePrefetchBuffer* prefetch_buffer = + (reinterpret_cast(iter.get())) + ->prefetch_buffer(); + { + // 1st Buffer Verification. + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, + block_handle); + ASSERT_EQ(buffer_len, 8192); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + // 2nd Buffer Verification. + prefetch_buffer->TEST_GetBufferOffsetandSize(1, buffer_offset, + buffer_len); + InternalKey ikey_tmp("00000540", 0, kTypeValue); + bbt->TEST_GetDataBlockHandle(read_options, ikey_tmp.Encode().ToString(), + block_handle); + ASSERT_EQ(buffer_len, 8192); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED), + 1); + } + + // Third prefetch ReadAsync (buffers will swap). + for (; kv_iter != kvmap.end() && iter->Valid(); kv_iter++) { + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + + if (iter->user_key() == "00000540") { + break; + } + + iter->Next(); + ASSERT_OK(iter->status()); + } + + { + // 1st Buffer Verification. + // curr buffer - 1. + prefetch_buffer->TEST_GetBufferOffsetandSize(1, buffer_offset, + buffer_len); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, + block_handle); + ASSERT_EQ(buffer_offset, block_handle.offset()); + ASSERT_EQ(buffer_len, 8192); + + // 2nd Buffer Verification. + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + InternalKey ikey_tmp("00000585", 0, kTypeValue); + bbt->TEST_GetDataBlockHandle(read_options, ikey_tmp.Encode().ToString(), + block_handle); + ASSERT_EQ(buffer_len, 4096); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED), + 1); + } + + // 4th Prefetch ReadAsync (buffers will swap). + for (; kv_iter != kvmap.end() && iter->Valid(); kv_iter++) { + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + + if (iter->user_key() == "00000585") { + break; + } + + iter->Next(); + ASSERT_OK(iter->status()); + } + + { + // 1st Buffer Verification. + // curr buffer - 0. + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, + block_handle); + ASSERT_EQ(buffer_offset, block_handle.offset()); + ASSERT_EQ(buffer_len, 4096); + + // 2nd Buffer Verification. + prefetch_buffer->TEST_GetBufferOffsetandSize(1, buffer_offset, + buffer_len); + InternalKey ikey_tmp("00000615", 0, kTypeValue); + bbt->TEST_GetDataBlockHandle(read_options, ikey_tmp.Encode().ToString(), + block_handle); + ASSERT_EQ(buffer_len, 4096); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED), + 1); + } + + // 5th Prefetch ReadAsync. + for (; kv_iter != kvmap.end() && iter->Valid(); kv_iter++) { + ASSERT_EQ(iter->key(), kv_iter->first); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + + if (iter->user_key() == "00000615") { + break; + } + + iter->Next(); + ASSERT_OK(iter->status()); + } + + { + // 1st Buffer Verification. + // curr_ - 1. + prefetch_buffer->TEST_GetBufferOffsetandSize(1, buffer_offset, + buffer_len); + ASSERT_EQ(buffer_len, 4096); + bbt->TEST_GetDataBlockHandle(read_options, kv_iter->first, + block_handle); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + // 2nd Buffer Verification. + prefetch_buffer->TEST_GetBufferOffsetandSize(0, buffer_offset, + buffer_len); + InternalKey ikey_tmp("00000630", 0, kTypeValue); + bbt->TEST_GetDataBlockHandle(read_options, ikey_tmp.Encode().ToString(), + block_handle); + ASSERT_EQ(buffer_len, 8192); + ASSERT_EQ(buffer_offset, block_handle.offset()); + + ASSERT_EQ(options.statistics->getAndResetTickerCount(READAHEAD_TRIMMED), + 0); + } + } + } + c.ResetTableReader(); +} + struct HitMissCountingCache : public CacheWrapper { using CacheWrapper::CacheWrapper; const char* Name() const override { return "HitMissCountingCache"; } @@ -3247,8 +3733,8 @@ TEST_P(BlockBasedTableTest, TracingMultiGetTest) { std::vector get_contexts; get_contexts.emplace_back( options.comparator, nullptr, nullptr, nullptr, GetContext::kNotFound, - ukeys[0], &values[0], nullptr, nullptr, nullptr, true, nullptr, nullptr, - nullptr, nullptr, nullptr, nullptr, get_id_offset); + ukeys[0], values.data(), nullptr, nullptr, nullptr, true, nullptr, + nullptr, nullptr, nullptr, nullptr, nullptr, get_id_offset); get_contexts.emplace_back( options.comparator, nullptr, nullptr, nullptr, GetContext::kNotFound, ukeys[1], &values[1], nullptr, nullptr, nullptr, true, nullptr, nullptr, @@ -3259,12 +3745,12 @@ TEST_P(BlockBasedTableTest, TracingMultiGetTest) { std::array statuses; autovector key_context; key_context.emplace_back(/*ColumnFamilyHandle omitted*/ nullptr, ukeys[0], - &values[0], + values.data(), /*PinnableWideColumns omitted*/ nullptr, - /*timestamp omitted*/ nullptr, &statuses[0]); + /*timestamp omitted*/ nullptr, statuses.data()); key_context[0].ukey_without_ts = ukeys[0]; key_context[0].ikey = encoded_keys[0]; - key_context[0].get_context = &get_contexts[0]; + key_context[0].get_context = get_contexts.data(); key_context.emplace_back(/*ColumnFamilyHandle omitted*/ nullptr, ukeys[1], &values[1], /*PinnableWideColumns omitted*/ nullptr, @@ -3303,8 +3789,8 @@ TEST_P(BlockBasedTableTest, TracingMultiGetTest) { record.block_type = TraceType::kBlockTraceFilterBlock; expected_records.push_back(record); } - // Then we should have three records for one index, one filter, and one data - // block access. (The two keys share a data block.) + // Then we should have three records for one index, one filter, and one + // data block access. (The two keys share a data block.) record.get_id = get_id_offset; record.block_type = TraceType::kBlockTraceFilterBlock; record.caller = TableReaderCaller::kUserMultiGet; @@ -3420,8 +3906,8 @@ TEST_P(BlockBasedTableTest, TracingIterator) { record.is_cache_hit = false; expected_records.push_back(record); expected_records.push_back(record); - // When we iterate this file for the second time, we should observe all cache - // hits. + // When we iterate this file for the second time, we should observe all + // cache hits. record.block_type = TraceType::kBlockTraceIndexBlock; record.is_cache_hit = true; expected_records.push_back(record); @@ -3495,8 +3981,8 @@ class BlockCachePropertiesSnapshot { int64_t block_cache_bytes_write = 0; }; -// Make sure, by default, index/filter blocks were pre-loaded (meaning we won't -// use block cache to store them). +// Make sure, by default, index/filter blocks were pre-loaded (meaning we +// won't use block cache to store them). TEST_P(BlockBasedTableTest, BlockCacheDisabledTest) { Options options; options.create_if_missing = true; @@ -3742,7 +4228,8 @@ void ValidateBlockRestartInterval(int value, int expected) { } TEST_P(BlockBasedTableTest, InvalidOptions) { - // invalid values for block_size_deviation (<0 or >100) are silently set to 0 + // invalid values for block_size_deviation (<0 or >100) are silently set to + // 0 ValidateBlockSizeDeviation(-10, 0); ValidateBlockSizeDeviation(-1, 0); ValidateBlockSizeDeviation(0, 0); @@ -3850,8 +4337,8 @@ TEST_P(BlockBasedTableTest, BlockReadCountTest) { TEST_P(BlockBasedTableTest, BlockCacheLeak) { // Check that when we reopen a table we don't lose access to blocks already - // in the cache. This test checks whether the Table actually makes use of the - // unique ID from the file. + // in the cache. This test checks whether the Table actually makes use of + // the unique ID from the file. Options opt; std::unique_ptr ikc; @@ -4175,7 +4662,6 @@ TEST_F(PlainTableTest, Crc32cFileChecksum) { EXPECT_STREQ(f.GetFileChecksum().c_str(), checksum.c_str()); } - TEST_F(GeneralTableTest, ApproximateOffsetOfPlain) { TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); c.Add("k01", "hello"); @@ -4304,7 +4790,8 @@ TEST_F(GeneralTableTest, ApproximateKeyAnchors) { std::vector anchors; ASSERT_OK(c.GetTableReader()->ApproximateKeyAnchors(ReadOptions(), anchors)); - // The target is 128 anchors. But in reality it can be slightly more or fewer. + // The target is 128 anchors. But in reality it can be slightly more or + // fewer. ASSERT_GT(anchors.size(), 120); ASSERT_LT(anchors.size(), 140); @@ -4472,11 +4959,12 @@ TEST(TableTest, FooterTests) { BlockHandle index(data_size + 5, index_size); BlockHandle meta_index(data_size + index_size + 2 * 5, metaindex_size); uint64_t footer_offset = data_size + metaindex_size + index_size + 3 * 5; + uint32_t base_context_checksum = 123456789; { // legacy block based FooterBuilder footer; - footer.Build(kBlockBasedTableMagicNumber, /* format_version */ 0, - footer_offset, kCRC32c, meta_index, index); + ASSERT_OK(footer.Build(kBlockBasedTableMagicNumber, /* format_version */ 0, + footer_offset, kCRC32c, meta_index, index)); Footer decoded_footer; ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset)); ASSERT_EQ(decoded_footer.table_magic_number(), kBlockBasedTableMagicNumber); @@ -4486,6 +4974,7 @@ TEST(TableTest, FooterTests) { ASSERT_EQ(decoded_footer.index_handle().offset(), index.offset()); ASSERT_EQ(decoded_footer.index_handle().size(), index.size()); ASSERT_EQ(decoded_footer.format_version(), 0U); + ASSERT_EQ(decoded_footer.base_context_checksum(), 0U); ASSERT_EQ(decoded_footer.GetBlockTrailerSize(), 5U); // Ensure serialized with legacy magic ASSERT_EQ( @@ -4495,9 +4984,11 @@ TEST(TableTest, FooterTests) { // block based, various checksums, various versions for (auto t : GetSupportedChecksums()) { for (uint32_t fv = 1; IsSupportedFormatVersion(fv); ++fv) { + uint32_t maybe_bcc = + FormatVersionUsesContextChecksum(fv) ? base_context_checksum : 0U; FooterBuilder footer; - footer.Build(kBlockBasedTableMagicNumber, fv, footer_offset, t, - meta_index, index); + ASSERT_OK(footer.Build(kBlockBasedTableMagicNumber, fv, footer_offset, t, + meta_index, index, maybe_bcc)); Footer decoded_footer; ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset)); ASSERT_EQ(decoded_footer.table_magic_number(), @@ -4506,18 +4997,44 @@ TEST(TableTest, FooterTests) { ASSERT_EQ(decoded_footer.metaindex_handle().offset(), meta_index.offset()); ASSERT_EQ(decoded_footer.metaindex_handle().size(), meta_index.size()); - ASSERT_EQ(decoded_footer.index_handle().offset(), index.offset()); - ASSERT_EQ(decoded_footer.index_handle().size(), index.size()); + if (FormatVersionUsesIndexHandleInFooter(fv)) { + ASSERT_EQ(decoded_footer.index_handle().offset(), index.offset()); + ASSERT_EQ(decoded_footer.index_handle().size(), index.size()); + } ASSERT_EQ(decoded_footer.format_version(), fv); ASSERT_EQ(decoded_footer.GetBlockTrailerSize(), 5U); + + if (FormatVersionUsesContextChecksum(fv)) { + ASSERT_EQ(decoded_footer.base_context_checksum(), + base_context_checksum); + + // Bad offset should fail footer checksum + decoded_footer = Footer(); + ASSERT_NOK( + decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset - 1)); + } else { + ASSERT_EQ(decoded_footer.base_context_checksum(), 0U); + } + + // Too big metaindex size should also fail encoding only in new footer + uint64_t big_metaindex_size = 0x100000007U; + uint64_t big_footer_offset = + data_size + big_metaindex_size + index_size + 3 * 5; + BlockHandle big_metaindex = + BlockHandle(data_size + index_size + 2 * 5, big_metaindex_size); + ASSERT_NE(footer + .Build(kBlockBasedTableMagicNumber, fv, big_footer_offset, + t, big_metaindex, index, maybe_bcc) + .ok(), + FormatVersionUsesContextChecksum(fv)); } } { // legacy plain table FooterBuilder footer; - footer.Build(kPlainTableMagicNumber, /* format_version */ 0, footer_offset, - kNoChecksum, meta_index); + ASSERT_OK(footer.Build(kPlainTableMagicNumber, /* format_version */ 0, + footer_offset, kNoChecksum, meta_index)); Footer decoded_footer; ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset)); ASSERT_EQ(decoded_footer.table_magic_number(), kPlainTableMagicNumber); @@ -4536,8 +5053,8 @@ TEST(TableTest, FooterTests) { { // xxhash plain table (not currently used) FooterBuilder footer; - footer.Build(kPlainTableMagicNumber, /* format_version */ 1, footer_offset, - kxxHash, meta_index); + ASSERT_OK(footer.Build(kPlainTableMagicNumber, /* format_version */ 1, + footer_offset, kxxHash, meta_index)); Footer decoded_footer; ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset)); ASSERT_EQ(decoded_footer.table_magic_number(), kPlainTableMagicNumber); @@ -4631,14 +5148,15 @@ TEST_P(IndexBlockRestartIntervalTest, IndexBlockRestartInterval) { class PrefixTest : public testing::Test { public: PrefixTest() : testing::Test() {} - ~PrefixTest() override {} + ~PrefixTest() override = default; }; namespace { // A simple PrefixExtractor that only works for test PrefixAndWholeKeyTest class TestPrefixExtractor : public ROCKSDB_NAMESPACE::SliceTransform { public: - ~TestPrefixExtractor() override{}; + ~TestPrefixExtractor() override = default; + ; const char* Name() const override { return "TestPrefixExtractor"; } ROCKSDB_NAMESPACE::Slice Transform( @@ -5211,9 +5729,13 @@ TEST_P(BlockBasedTableTest, PropertiesMetaBlockLast) { } } ASSERT_EQ(kPropertiesBlockName, key_at_max_offset); - // index handle is stored in footer rather than metaindex block, so need - // separate logic to verify it comes before properties block. - ASSERT_GT(max_offset, footer.index_handle().offset()); + if (FormatVersionUsesIndexHandleInFooter(footer.format_version())) { + // If index handle is stored in footer rather than metaindex block, + // need separate logic to verify it comes before properties block. + ASSERT_GT(max_offset, footer.index_handle().offset()); + } else { + ASSERT_TRUE(footer.index_handle().IsNull()); + } c.ResetTableReader(); } @@ -5362,16 +5884,13 @@ TEST_F(BBTTailPrefetchTest, FilePrefetchBufferMinOffset) { IOOptions opts; buffer.TryReadFromCache(opts, nullptr /* reader */, 500 /* offset */, 10 /* n */, nullptr /* result */, - nullptr /* status */, - Env::IO_TOTAL /* rate_limiter_priority */); + nullptr /* status */); buffer.TryReadFromCache(opts, nullptr /* reader */, 480 /* offset */, 10 /* n */, nullptr /* result */, - nullptr /* status */, - Env::IO_TOTAL /* rate_limiter_priority */); + nullptr /* status */); buffer.TryReadFromCache(opts, nullptr /* reader */, 490 /* offset */, 10 /* n */, nullptr /* result */, - nullptr /* status */, - Env::IO_TOTAL /* rate_limiter_priority */); + nullptr /* status */); ASSERT_EQ(480, buffer.min_offset_read()); } diff --git a/table/two_level_iterator.cc b/table/two_level_iterator.cc index 4b6634e5c..c66a94fb5 100644 --- a/table/two_level_iterator.cc +++ b/table/two_level_iterator.cc @@ -70,7 +70,9 @@ class TwoLevelIndexIterator : public InternalIteratorBase { private: void SaveError(const Status& s) { - if (status_.ok() && !s.ok()) status_ = s; + if (status_.ok() && !s.ok()) { + status_ = s; + } } void SkipEmptyDataBlocksForward(); void SkipEmptyDataBlocksBackward(); diff --git a/table/unique_id.cc b/table/unique_id.cc index fcdd75650..8bfa8bcfd 100644 --- a/table/unique_id.cc +++ b/table/unique_id.cc @@ -14,7 +14,7 @@ namespace ROCKSDB_NAMESPACE { std::string EncodeSessionId(uint64_t upper, uint64_t lower) { std::string db_session_id(20U, '\0'); - char *buf = &db_session_id[0]; + char *buf = db_session_id.data(); // Preserving `lower` is slightly tricky. 36^12 is slightly more than // 62 bits, so we use 12 chars plus the bottom two bits of one more. // (A tiny fraction of 20 digit strings go unused.) @@ -152,7 +152,7 @@ void ExternalUniqueIdToInternal(UniqueIdPtr in_out) { std::string EncodeUniqueIdBytes(UniqueIdPtr in) { std::string ret(in.extended ? 24U : 16U, '\0'); - EncodeFixed64(&ret[0], in.ptr[0]); + EncodeFixed64(ret.data(), in.ptr[0]); EncodeFixed64(&ret[8], in.ptr[1]); if (in.extended) { EncodeFixed64(&ret[16], in.ptr[2]); diff --git a/test_util/mock_time_env.h b/test_util/mock_time_env.h index 7834368e0..19bb9e76d 100644 --- a/test_util/mock_time_env.h +++ b/test_util/mock_time_env.h @@ -8,7 +8,11 @@ #include #include +#include "port/port.h" #include "rocksdb/system_clock.h" +#include "test_util/mock_time_env.h" +#include "test_util/sync_point.h" +#include "util/random.h" namespace ROCKSDB_NAMESPACE { @@ -65,6 +69,33 @@ class MockSystemClock : public SystemClockWrapper { current_time_us_.fetch_add(micros); } + virtual bool TimedWait(port::CondVar* cv, + std::chrono::microseconds deadline) override { + uint64_t now_micros = NowMicros(); + uint64_t deadline_micros = static_cast(deadline.count()); + uint64_t delay_micros; + if (deadline_micros > now_micros) { + delay_micros = deadline_micros - now_micros; + } else { + delay_micros = 0; + } + // To prevent slowdown, this `TimedWait()` is completely synthetic. First, + // it yields to coerce other threads to run while the lock is released. + // Second, it randomly selects between mocking an immediate wakeup and a + // timeout. + cv->GetMutex()->Unlock(); + std::this_thread::yield(); + bool mock_timeout = Random::GetTLSInstance()->OneIn(2); + if (mock_timeout) { + TEST_SYNC_POINT("MockSystemClock::TimedWait:UnlockedPreSleep"); + current_time_us_.fetch_add(delay_micros); + TEST_SYNC_POINT("MockSystemClock::TimedWait:UnlockedPostSleep1"); + TEST_SYNC_POINT("MockSystemClock::TimedWait:UnlockedPostSleep2"); + } + cv->GetMutex()->Lock(); + return mock_timeout; + } + // TODO: this is a workaround for the different behavior on different platform // for timedwait timeout. Ideally timedwait API should be moved to env. // details: PR #7101. diff --git a/test_util/secondary_cache_test_util.cc b/test_util/secondary_cache_test_util.cc index 1c62dc4ad..6f0bd3849 100644 --- a/test_util/secondary_cache_test_util.cc +++ b/test_util/secondary_cache_test_util.cc @@ -37,7 +37,8 @@ Status SaveToCallbackFail(Cache::ObjectPtr /*obj*/, size_t /*offset*/, return Status::NotSupported(); } -Status CreateCallback(const Slice& data, Cache::CreateContext* context, +Status CreateCallback(const Slice& data, CompressionType /*type*/, + CacheTier /*source*/, Cache::CreateContext* context, MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj, size_t* out_charge) { auto t = static_cast(context); diff --git a/test_util/secondary_cache_test_util.h b/test_util/secondary_cache_test_util.h index 1cfb454b5..5e2262a9c 100644 --- a/test_util/secondary_cache_test_util.h +++ b/test_util/secondary_cache_test_util.h @@ -42,12 +42,19 @@ class WithCacheType : public TestCreateContext { }; static constexpr auto kLRU = "lru"; - static constexpr auto kHyperClock = "hyper_clock"; + static constexpr auto kFixedHyperClock = "fixed_hyper_clock"; + static constexpr auto kAutoHyperClock = "auto_hyper_clock"; // For options other than capacity size_t estimated_value_size_ = 1; - virtual const std::string& Type() = 0; + virtual const std::string& Type() const = 0; + + static bool IsHyperClock(const std::string& type) { + return type == kFixedHyperClock || type == kAutoHyperClock; + } + + bool IsHyperClock() const { return IsHyperClock(Type()); } std::shared_ptr NewCache( size_t capacity, @@ -62,8 +69,11 @@ class WithCacheType : public TestCreateContext { } return lru_opts.MakeSharedCache(); } - if (type == kHyperClock) { - HyperClockCacheOptions hc_opts{capacity, estimated_value_size_}; + if (IsHyperClock(type)) { + HyperClockCacheOptions hc_opts{ + capacity, type == kFixedHyperClock ? estimated_value_size_ : 0}; + hc_opts.min_avg_entry_charge = + std::max(size_t{1}, estimated_value_size_ / 2); hc_opts.hash_seed = 0; // deterministic tests if (modify_opts_fn) { modify_opts_fn(hc_opts); @@ -105,14 +115,16 @@ class WithCacheType : public TestCreateContext { class WithCacheTypeParam : public WithCacheType, public testing::WithParamInterface { - const std::string& Type() override { return GetParam(); } + const std::string& Type() const override { return GetParam(); } }; constexpr auto kLRU = WithCacheType::kLRU; -constexpr auto kHyperClock = WithCacheType::kHyperClock; +constexpr auto kFixedHyperClock = WithCacheType::kFixedHyperClock; +constexpr auto kAutoHyperClock = WithCacheType::kAutoHyperClock; inline auto GetTestingCacheTypes() { - return testing::Values(std::string(kLRU), std::string(kHyperClock)); + return testing::Values(std::string(kLRU), std::string(kFixedHyperClock), + std::string(kAutoHyperClock)); } } // namespace secondary_cache_test_util diff --git a/test_util/testutil.cc b/test_util/testutil.cc index d958cb0cd..ce221e79b 100644 --- a/test_util/testutil.cc +++ b/test_util/testutil.cc @@ -39,7 +39,10 @@ namespace test { const uint32_t kDefaultFormatVersion = BlockBasedTableOptions().format_version; const std::set kFooterFormatVersionsToTest{ + // Non-legacy, before big footer changes 5U, + // After big footer changes + 6U, // In case any interesting future changes kDefaultFormatVersion, kLatestFormatVersion, @@ -91,7 +94,9 @@ bool ShouldPersistUDT(const UserDefinedTimestampTestMode& test_mode) { extern Slice CompressibleString(Random* rnd, double compressed_fraction, int len, std::string* dst) { int raw = static_cast(len * compressed_fraction); - if (raw < 1) raw = 1; + if (raw < 1) { + raw = 1; + } std::string raw_data = rnd->RandomBinaryString(raw); // Duplicate the random data until we have filled "len" bytes @@ -106,7 +111,7 @@ extern Slice CompressibleString(Random* rnd, double compressed_fraction, namespace { class Uint64ComparatorImpl : public Comparator { public: - Uint64ComparatorImpl() {} + Uint64ComparatorImpl() = default; const char* Name() const override { return "rocksdb.Uint64Comparator"; } @@ -128,11 +133,9 @@ class Uint64ComparatorImpl : public Comparator { } void FindShortestSeparator(std::string* /*start*/, - const Slice& /*limit*/) const override { - return; - } + const Slice& /*limit*/) const override {} - void FindShortSuccessor(std::string* /*key*/) const override { return; } + void FindShortSuccessor(std::string* /*key*/) const override {} }; } // namespace @@ -150,6 +153,16 @@ const Comparator* BytewiseComparatorWithU64TsWrapper() { return user_comparator; } +const Comparator* ReverseBytewiseComparatorWithU64TsWrapper() { + ConfigOptions config_options; + const Comparator* user_comparator = nullptr; + Status s = Comparator::CreateFromString( + config_options, "rocksdb.ReverseBytewiseComparator.u64ts", + &user_comparator); + s.PermitUncheckedError(); + return user_comparator; +} + void CorruptKeyType(InternalKey* ikey) { std::string keystr = ikey->Encode().ToString(); keystr[keystr.size() - 8] = kTypeLogData; @@ -619,7 +632,7 @@ class SpecialMemTableRep : public MemTableRep { return memtable_->GetIterator(arena); } - virtual ~SpecialMemTableRep() override {} + virtual ~SpecialMemTableRep() override = default; private: std::unique_ptr memtable_; @@ -634,7 +647,7 @@ class SpecialSkipListFactory : public MemTableRepFactory { .AddNumber(":"), [](const std::string& uri, std::unique_ptr* guard, std::string* /* errmsg */) { - auto colon = uri.find(":"); + auto colon = uri.find(':'); if (colon != std::string::npos) { auto count = ParseInt(uri.substr(colon + 1)); guard->reset(new SpecialSkipListFactory(count)); diff --git a/test_util/testutil.h b/test_util/testutil.h index c40fcdcb0..eca1ff794 100644 --- a/test_util/testutil.h +++ b/test_util/testutil.h @@ -132,6 +132,9 @@ extern const Comparator* Uint64Comparator(); // A wrapper api for getting the ComparatorWithU64Ts extern const Comparator* BytewiseComparatorWithU64TsWrapper(); +// A wrapper api for getting the ComparatorWithU64Ts +extern const Comparator* ReverseBytewiseComparatorWithU64TsWrapper(); + class StringSink : public FSWritableFile { public: std::string contents_; diff --git a/tools/block_cache_analyzer/block_cache_pysim.py b/tools/block_cache_analyzer/block_cache_pysim.py index 67307df53..3962f37eb 100644 --- a/tools/block_cache_analyzer/block_cache_pysim.py +++ b/tools/block_cache_analyzer/block_cache_pysim.py @@ -492,7 +492,7 @@ def write_policy_ratio_timeline( file.write(row + "\n") -class Policy(object): +class Policy: """ A policy maintains a set of evicted keys. It returns a reward of one to itself if it has not evicted a missing key. Otherwise, it gives itself 0 @@ -654,7 +654,7 @@ def policy_name(self): return "cc" -class Cache(object): +class Cache: """ This is the base class for the implementations of alternative cache replacement policies. @@ -1310,7 +1310,7 @@ def _should_admit(self, trace_record, key, hash, value_size): return True -class Deque(object): +class Deque: """A Deque class facilitates the implementation of LRU and ARC.""" def __init__(self): diff --git a/tools/block_cache_analyzer/block_cache_trace_analyzer.cc b/tools/block_cache_analyzer/block_cache_trace_analyzer.cc index f2d4f05be..e6473191d 100644 --- a/tools/block_cache_analyzer/block_cache_trace_analyzer.cc +++ b/tools/block_cache_analyzer/block_cache_trace_analyzer.cc @@ -577,7 +577,7 @@ void BlockCacheTraceAnalyzer::WriteSkewness( std::map> label_bucket_naccesses; std::vector> pairs; for (auto const& itr : label_naccesses) { - pairs.push_back(itr); + pairs.emplace_back(itr); } // Sort in descending order. sort(pairs.begin(), pairs.end(), diff --git a/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc b/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc index 174565641..77a6d1b2b 100644 --- a/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +++ b/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc @@ -492,7 +492,7 @@ TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) { ASSERT_EQ(20, ParseDouble(percent)); } ASSERT_EQ(expected_callers.size(), callers.size()); - for (auto caller : callers) { + for (const auto& caller : callers) { ASSERT_TRUE(expected_callers.find(caller) != expected_callers.end()); } ASSERT_OK(env_->DeleteFile(percent_access_summary_file)); @@ -504,7 +504,7 @@ TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) { std::string caller; ASSERT_TRUE(getline(analyzing_callers, caller, ',')); std::vector breakdowns{"level", "bt"}; - for (auto breakdown : breakdowns) { + for (const auto& breakdown : breakdowns) { const std::string file_name = test_path_ + "/" + caller + "_" + breakdown + "_percentage_of_accesses_summary"; @@ -554,7 +554,7 @@ TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) { } for (auto const& access_type : access_types) { std::vector block_types{"Index", "Data", "Filter"}; - for (auto block_type : block_types) { + for (const auto& block_type : block_types) { // Validate reuse block timeline. const std::string reuse_blocks_timeline = test_path_ + "/" + block_type + "_" + access_type + diff --git a/tools/check_format_compatible.sh b/tools/check_format_compatible.sh index 6403b2675..93b51a9b9 100755 --- a/tools/check_format_compatible.sh +++ b/tools/check_format_compatible.sh @@ -125,7 +125,7 @@ EOF # To check for DB forward compatibility with loading options (old version # reading data from new), as well as backward compatibility -declare -a db_forward_with_options_refs=("6.27.fb" "6.28.fb" "6.29.fb" "7.0.fb" "7.1.fb" "7.2.fb" "7.3.fb" "7.4.fb" "7.5.fb" "7.6.fb" "7.7.fb" "7.8.fb" "7.9.fb" "7.10.fb" "8.0.fb" "8.1.fb" "8.2.fb" "8.3.fb" "8.4.fb") +declare -a db_forward_with_options_refs=("6.27.fb" "6.28.fb" "6.29.fb" "7.0.fb" "7.1.fb" "7.2.fb" "7.3.fb" "7.4.fb" "7.5.fb" "7.6.fb" "7.7.fb" "7.8.fb" "7.9.fb" "7.10.fb" "8.0.fb" "8.1.fb" "8.2.fb" "8.3.fb" "8.4.fb" "8.5.fb" "8.6.fb" "8.7.fb" "8.8.fb" "8.9.fb") # To check for DB forward compatibility without loading options (in addition # to the "with loading options" set), as well as backward compatibility declare -a db_forward_no_options_refs=() # N/A at the moment diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 69d35ab2d..e177934b0 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -15,9 +15,10 @@ #include #endif #include -#include -#include #include + +#include +#include #ifdef __APPLE__ #include #include @@ -602,12 +603,17 @@ DEFINE_uint32( "compress_format_version == 2 -- decompressed size is included" " in the block header in varint32 format."); -DEFINE_bool(use_tiered_volatile_cache, false, +DEFINE_bool(use_tiered_cache, false, "If use_compressed_secondary_cache is true and " "use_tiered_volatile_cache is true, then allocate a tiered cache " "that distributes cache reservations proportionally over both " "the caches."); +DEFINE_string( + tiered_adm_policy, "auto", + "Admission policy to use for the secondary cache(s) in the tiered cache. " + "Allowed values are auto, placeholder, allow_cache_hits, and three_queue."); + DEFINE_int64(simcache_size, -1, "Number of bytes to use as a simcache of " "uncompressed data. Nagative value disables simcache."); @@ -718,7 +724,9 @@ DEFINE_int32(file_opening_threads, "If open_files is set to -1, this option set the number of " "threads that will be used to open files during DB::Open()"); -DEFINE_int32(compaction_readahead_size, 0, "Compaction readahead size"); +DEFINE_uint64(compaction_readahead_size, + ROCKSDB_NAMESPACE::Options().compaction_readahead_size, + "Compaction readahead size"); DEFINE_int32(log_readahead_size, 0, "WAL and manifest readahead size"); @@ -1243,32 +1251,54 @@ DEFINE_uint64( "num_file_reads_for_auto_readahead indicates after how many sequential " "reads into that file internal auto prefetching should be start."); +DEFINE_bool( + auto_readahead_size, false, + "When set true, RocksDB does auto tuning of readahead size during Scans"); + static enum ROCKSDB_NAMESPACE::CompressionType StringToCompressionType( const char* ctype) { assert(ctype); - if (!strcasecmp(ctype, "none")) + if (!strcasecmp(ctype, "none")) { return ROCKSDB_NAMESPACE::kNoCompression; - else if (!strcasecmp(ctype, "snappy")) + } else if (!strcasecmp(ctype, "snappy")) { return ROCKSDB_NAMESPACE::kSnappyCompression; - else if (!strcasecmp(ctype, "zlib")) + } else if (!strcasecmp(ctype, "zlib")) { return ROCKSDB_NAMESPACE::kZlibCompression; - else if (!strcasecmp(ctype, "bzip2")) + } else if (!strcasecmp(ctype, "bzip2")) { return ROCKSDB_NAMESPACE::kBZip2Compression; - else if (!strcasecmp(ctype, "lz4")) + } else if (!strcasecmp(ctype, "lz4")) { return ROCKSDB_NAMESPACE::kLZ4Compression; - else if (!strcasecmp(ctype, "lz4hc")) + } else if (!strcasecmp(ctype, "lz4hc")) { return ROCKSDB_NAMESPACE::kLZ4HCCompression; - else if (!strcasecmp(ctype, "xpress")) + } else if (!strcasecmp(ctype, "xpress")) { return ROCKSDB_NAMESPACE::kXpressCompression; - else if (!strcasecmp(ctype, "zstd")) + } else if (!strcasecmp(ctype, "zstd")) { return ROCKSDB_NAMESPACE::kZSTD; - else { + } else { fprintf(stderr, "Cannot parse compression type '%s'\n", ctype); exit(1); } } +static enum ROCKSDB_NAMESPACE::TieredAdmissionPolicy StringToAdmissionPolicy( + const char* policy) { + assert(policy); + + if (!strcasecmp(policy, "auto")) { + return ROCKSDB_NAMESPACE::kAdmPolicyAuto; + } else if (!strcasecmp(policy, "placeholder")) { + return ROCKSDB_NAMESPACE::kAdmPolicyPlaceholder; + } else if (!strcasecmp(policy, "allow_cache_hits")) { + return ROCKSDB_NAMESPACE::kAdmPolicyAllowCacheHits; + } else if (!strcasecmp(policy, "three_queue")) { + return ROCKSDB_NAMESPACE::kAdmPolicyThreeQueue; + } else { + fprintf(stderr, "Cannot parse admission policy %s\n", policy); + exit(1); + } +} + static std::string ColumnFamilyName(size_t i) { if (i == 0) { return ROCKSDB_NAMESPACE::kDefaultColumnFamilyName; @@ -1777,12 +1807,13 @@ static enum DistributionType FLAGS_value_size_distribution_type_e = kFixed; static enum DistributionType StringToDistributionType(const char* ctype) { assert(ctype); - if (!strcasecmp(ctype, "fixed")) + if (!strcasecmp(ctype, "fixed")) { return kFixed; - else if (!strcasecmp(ctype, "uniform")) + } else if (!strcasecmp(ctype, "uniform")) { return kUniform; - else if (!strcasecmp(ctype, "normal")) + } else if (!strcasecmp(ctype, "normal")) { return kNormal; + } fprintf(stdout, "Cannot parse distribution type '%s'\n", ctype); exit(1); @@ -1792,7 +1823,7 @@ class BaseDistribution { public: BaseDistribution(unsigned int _min, unsigned int _max) : min_value_size_(_min), max_value_size_(_max) {} - virtual ~BaseDistribution() {} + virtual ~BaseDistribution() = default; unsigned int Generate() { auto val = Get(); @@ -1909,7 +1940,9 @@ class RandomGenerator { }; static void AppendWithSpace(std::string* str, Slice msg) { - if (msg.empty()) return; + if (msg.empty()) { + return; + } if (!str->empty()) { str->push_back(' '); } @@ -2163,7 +2196,9 @@ class Stats { } void Merge(const Stats& other) { - if (other.exclude_from_merge_) return; + if (other.exclude_from_merge_) { + return; + } for (auto it = other.hist_.begin(); it != other.hist_.end(); ++it) { auto this_it = hist_.find(it->first); @@ -2177,11 +2212,17 @@ class Stats { done_ += other.done_; bytes_ += other.bytes_; seconds_ += other.seconds_; - if (other.start_ < start_) start_ = other.start_; - if (other.finish_ > finish_) finish_ = other.finish_; + if (other.start_ < start_) { + start_ = other.start_; + } + if (other.finish_ > finish_) { + finish_ = other.finish_; + } // Just keep the messages from one thread. - if (message_.empty()) message_ = other.message_; + if (message_.empty()) { + message_ = other.message_; + } } void Stop() { @@ -2260,20 +2301,21 @@ class Stats { done_ += num_ops; if (done_ >= next_report_ && FLAGS_progress_reports) { if (!FLAGS_stats_interval) { - if (next_report_ < 1000) + if (next_report_ < 1000) { next_report_ += 100; - else if (next_report_ < 5000) + } else if (next_report_ < 5000) { next_report_ += 500; - else if (next_report_ < 10000) + } else if (next_report_ < 10000) { next_report_ += 1000; - else if (next_report_ < 50000) + } else if (next_report_ < 50000) { next_report_ += 5000; - else if (next_report_ < 100000) + } else if (next_report_ < 100000) { next_report_ += 10000; - else if (next_report_ < 500000) + } else if (next_report_ < 500000) { next_report_ += 50000; - else + } else { next_report_ += 100000; + } fprintf(stderr, "... finished %" PRIu64 " ops%30s\r", done_, ""); } else { uint64_t now = clock_->NowMicros(); @@ -2305,8 +2347,9 @@ class Stats { if (db_with_cfh && db_with_cfh->num_created.load()) { for (size_t i = 0; i < db_with_cfh->num_created.load(); ++i) { if (db->GetProperty(db_with_cfh->cfh[i], "rocksdb.cfstats", - &stats)) + &stats)) { fprintf(stderr, "%s\n", stats.c_str()); + } if (FLAGS_show_table_properties) { for (int level = 0; level < FLAGS_num_levels; ++level) { if (db->GetProperty( @@ -2364,7 +2407,9 @@ class Stats { void Report(const Slice& name) { // Pretend at least one op was done in case we are running a benchmark // that does not call FinishedOps(). - if (done_ < 1) done_ = 1; + if (done_ < 1) { + done_ = 1; + } std::string extra; double elapsed = (finish_ - start_) * 1e-6; @@ -2629,7 +2674,9 @@ class Duration { int64_t GetStage() { return std::min(ops_, max_ops_ - 1) / ops_per_stage_; } bool Done(int64_t increment) { - if (increment <= 0) increment = 1; // avoid Done(0) and infinite loops + if (increment <= 0) { + increment = 1; // avoid Done(0) and infinite loops + } ops_ += increment; if (max_seconds_) { @@ -2696,7 +2743,7 @@ class Benchmark { no_auto_recovery_(false), recovery_complete_(false) {} - ~ErrorHandlerListener() override {} + ~ErrorHandlerListener() override = default; const char* Name() const override { return kClassName(); } static const char* kClassName() { return "ErrorHandlerListener"; } @@ -2834,7 +2881,7 @@ class Benchmark { std::string input_str(len, 'y'); std::string compressed; CompressionOptions opts; - CompressionContext context(FLAGS_compression_type_e); + CompressionContext context(FLAGS_compression_type_e, opts); CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), FLAGS_compression_type_e, FLAGS_sample_for_compression); @@ -3016,6 +3063,7 @@ class Benchmark { static std::shared_ptr NewCache(int64_t capacity) { CompressedSecondaryCacheOptions secondary_cache_opts; + TieredAdmissionPolicy adm_policy = TieredAdmissionPolicy::kAdmPolicyAuto; bool use_tiered_cache = false; if (capacity <= 0) { return nullptr; @@ -3032,28 +3080,67 @@ class Benchmark { FLAGS_compressed_secondary_cache_compression_type_e; secondary_cache_opts.compress_format_version = FLAGS_compressed_secondary_cache_compress_format_version; - if (FLAGS_use_tiered_volatile_cache) { + if (FLAGS_use_tiered_cache) { use_tiered_cache = true; + adm_policy = StringToAdmissionPolicy(FLAGS_tiered_adm_policy.c_str()); + } + } + if (!FLAGS_secondary_cache_uri.empty()) { + if (!use_tiered_cache && FLAGS_use_compressed_secondary_cache) { + fprintf( + stderr, + "Cannot specify both --secondary_cache_uri and " + "--use_compressed_secondary_cache when using a non-tiered cache\n"); + exit(1); + } + Status s = SecondaryCache::CreateFromString( + ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache); + if (secondary_cache == nullptr) { + fprintf(stderr, + "No secondary cache registered matching string: %s status=%s\n", + FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str()); + exit(1); } } + + std::shared_ptr block_cache; if (FLAGS_cache_type == "clock_cache") { fprintf(stderr, "Old clock cache implementation has been removed.\n"); exit(1); - } else if (FLAGS_cache_type == "hyper_clock_cache") { - HyperClockCacheOptions hcco{ - static_cast(capacity), - static_cast(FLAGS_block_size) /*estimated_entry_charge*/, - FLAGS_cache_numshardbits}; - hcco.hash_seed = GetCacheHashSeed(); + } else if (EndsWith(FLAGS_cache_type, "hyper_clock_cache")) { + size_t estimated_entry_charge; + if (FLAGS_cache_type == "fixed_hyper_clock_cache" || + FLAGS_cache_type == "hyper_clock_cache") { + estimated_entry_charge = FLAGS_block_size; + } else if (FLAGS_cache_type == "auto_hyper_clock_cache") { + estimated_entry_charge = 0; + } else { + fprintf(stderr, "Cache type not supported."); + exit(1); + } + HyperClockCacheOptions opts(FLAGS_cache_size, estimated_entry_charge, + FLAGS_cache_numshardbits); + opts.hash_seed = GetCacheHashSeed(); if (use_tiered_cache) { - TieredVolatileCacheOptions opts; - hcco.capacity += secondary_cache_opts.capacity; - opts.cache_type = PrimaryCacheType::kCacheTypeHCC; - opts.cache_opts = &hcco; - opts.comp_cache_opts = secondary_cache_opts; - return NewTieredVolatileCache(opts); + TieredCacheOptions tiered_opts; + tiered_opts.cache_type = PrimaryCacheType::kCacheTypeHCC; + tiered_opts.cache_opts = &opts; + tiered_opts.total_capacity = + opts.capacity + secondary_cache_opts.capacity; + tiered_opts.compressed_secondary_ratio = + secondary_cache_opts.capacity * 1.0 / tiered_opts.total_capacity; + tiered_opts.comp_cache_opts = secondary_cache_opts; + tiered_opts.nvm_sec_cache = secondary_cache; + tiered_opts.adm_policy = adm_policy; + block_cache = NewTieredCache(tiered_opts); } else { - return hcco.MakeSharedCache(); + if (!FLAGS_secondary_cache_uri.empty()) { + opts.secondary_cache = secondary_cache; + } else if (FLAGS_use_compressed_secondary_cache) { + opts.secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + } + block_cache = opts.MakeSharedCache(); } } else if (FLAGS_cache_type == "lru_cache") { LRUCacheOptions opts( @@ -3062,36 +3149,37 @@ class Benchmark { GetCacheAllocator(), kDefaultToAdaptiveMutex, kDefaultCacheMetadataChargePolicy, FLAGS_cache_low_pri_pool_ratio); opts.hash_seed = GetCacheHashSeed(); - if (!FLAGS_secondary_cache_uri.empty()) { - Status s = SecondaryCache::CreateFromString( - ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache); - if (secondary_cache == nullptr) { - fprintf( - stderr, - "No secondary cache registered matching string: %s status=%s\n", - FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str()); - exit(1); - } - opts.secondary_cache = secondary_cache; - } else if (FLAGS_use_compressed_secondary_cache && !use_tiered_cache) { - opts.secondary_cache = - NewCompressedSecondaryCache(secondary_cache_opts); - } - if (use_tiered_cache) { - TieredVolatileCacheOptions tiered_opts; - opts.capacity += secondary_cache_opts.capacity; + TieredCacheOptions tiered_opts; tiered_opts.cache_type = PrimaryCacheType::kCacheTypeLRU; tiered_opts.cache_opts = &opts; + tiered_opts.total_capacity = + opts.capacity + secondary_cache_opts.capacity; + tiered_opts.compressed_secondary_ratio = + secondary_cache_opts.capacity * 1.0 / tiered_opts.total_capacity; tiered_opts.comp_cache_opts = secondary_cache_opts; - return NewTieredVolatileCache(tiered_opts); + tiered_opts.nvm_sec_cache = secondary_cache; + tiered_opts.adm_policy = adm_policy; + block_cache = NewTieredCache(tiered_opts); } else { - return opts.MakeSharedCache(); + if (!FLAGS_secondary_cache_uri.empty()) { + opts.secondary_cache = secondary_cache; + } else if (FLAGS_use_compressed_secondary_cache) { + opts.secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + } + block_cache = opts.MakeSharedCache(); } } else { fprintf(stderr, "Cache type not supported."); exit(1); } + + if (!block_cache) { + fprintf(stderr, "Unable to allocate block cache\n"); + exit(1); + } + return block_cache; } public: @@ -3360,6 +3448,7 @@ class Benchmark { read_options_.adaptive_readahead = FLAGS_adaptive_readahead; read_options_.async_io = FLAGS_async_io; read_options_.optimize_multiget_for_io = FLAGS_optimize_multiget_for_io; + read_options_.auto_readahead_size = FLAGS_auto_readahead_size; void (Benchmark::*method)(ThreadState*) = nullptr; void (Benchmark::*post_process_method)() = nullptr; @@ -3593,6 +3682,8 @@ class Benchmark { } else if (name == "block_cache_entry_stats") { // DB::Properties::kBlockCacheEntryStats PrintStats("rocksdb.block-cache-entry-stats"); + } else if (name == "cache_report_problems") { + CacheReportProblems(); } else if (name == "stats") { PrintStats("rocksdb.stats"); } else if (name == "resetstats") { @@ -3983,7 +4074,9 @@ class Benchmark { count++; thread->stats.FinishedOps(nullptr, nullptr, 1, kOthers); } - if (ptr == nullptr) exit(1); // Disable unused variable warning. + if (ptr == nullptr) { + exit(1); // Disable unused variable warning. + } } void Compress(ThreadState* thread) { @@ -3994,7 +4087,8 @@ class Benchmark { bool ok = true; std::string compressed; CompressionOptions opts; - CompressionContext context(FLAGS_compression_type_e); + opts.level = FLAGS_compression_level; + CompressionContext context(FLAGS_compression_type_e, opts); CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), FLAGS_compression_type_e, FLAGS_sample_for_compression); @@ -4023,8 +4117,10 @@ class Benchmark { Slice input = gen.Generate(FLAGS_block_size); std::string compressed; - CompressionContext compression_ctx(FLAGS_compression_type_e); CompressionOptions compression_opts; + compression_opts.level = FLAGS_compression_level; + CompressionContext compression_ctx(FLAGS_compression_type_e, + compression_opts); CompressionInfo compression_info( compression_opts, compression_ctx, CompressionDict::GetEmptyDict(), FLAGS_compression_type_e, FLAGS_sample_for_compression); @@ -4152,7 +4248,7 @@ class Benchmark { } } if (FLAGS_use_stderr_info_logger) { - options.info_log.reset(new StderrLogger()); + options.info_log = std::make_shared(); } options.memtable_huge_page_size = FLAGS_memtable_use_huge_page ? 2048 : 0; options.memtable_prefix_bloom_size_ratio = FLAGS_memtable_bloom_size_ratio; @@ -4760,8 +4856,8 @@ class Benchmark { } std::vector column_families; for (size_t i = 0; i < num_hot; i++) { - column_families.push_back(ColumnFamilyDescriptor( - ColumnFamilyName(i), ColumnFamilyOptions(options))); + column_families.emplace_back(ColumnFamilyName(i), + ColumnFamilyOptions(options)); } std::vector cfh_idx_to_prob; if (!FLAGS_column_family_distribution.empty()) { @@ -5584,7 +5680,7 @@ class Benchmark { auto total_size = meta.levels[0].size; if (total_size >= db->GetOptions().compaction_options_fifo.max_table_files_size) { - for (auto file_meta : meta.levels[0].files) { + for (const auto& file_meta : meta.levels[0].files) { file_names.emplace_back(file_meta.name); } break; @@ -5635,7 +5731,7 @@ class Benchmark { SequenceNumber sorted_run_largest_seqno = 0; std::string sorted_run_smallest_key, sorted_run_largest_key; bool first_key = true; - for (auto fileMeta : sorted_runs[k][i]) { + for (const auto& fileMeta : sorted_runs[k][i]) { sorted_run_smallest_seqno = std::min(sorted_run_smallest_seqno, fileMeta.smallest_seqno); sorted_run_largest_seqno = @@ -5656,7 +5752,7 @@ class Benchmark { (compaction_style == kCompactionStyleUniversal && level > 0)) { SequenceNumber level_smallest_seqno = kMaxSequenceNumber; SequenceNumber level_largest_seqno = 0; - for (auto fileMeta : meta.levels[level].files) { + for (const auto& fileMeta : meta.levels[level].files) { level_smallest_seqno = std::min(level_smallest_seqno, fileMeta.smallest_seqno); level_largest_seqno = @@ -5743,6 +5839,7 @@ class Benchmark { options.adaptive_readahead = FLAGS_adaptive_readahead; options.async_io = FLAGS_async_io; + options.auto_readahead_size = FLAGS_auto_readahead_size; Iterator* iter = db->NewIterator(options); int64_t i = 0; @@ -6177,8 +6274,8 @@ class Benchmark { GenerateKeyFromInt(lkey, FLAGS_num, &lkeys[i]); GenerateKeyFromInt(rkey, FLAGS_num, &rkeys[i]); } - db->GetApproximateSizes(&ranges[0], static_cast(entries_per_batch_), - &sizes[0]); + db->GetApproximateSizes( + ranges.data(), static_cast(entries_per_batch_), sizes.data()); num_sizes += entries_per_batch_; for (int64_t size : sizes) { size_sum += size; @@ -6231,8 +6328,8 @@ class Benchmark { std::vector ratio_; int range_; - QueryDecider() {} - ~QueryDecider() {} + QueryDecider() = default; + ~QueryDecider() = default; Status Initiate(std::vector ratio_input) { int range_max = 1000; @@ -7575,7 +7672,9 @@ class Benchmark { thread->stats.FinishedOps(nullptr, db, 1, kMerge); } else { Status s = db->Get(read_options_, key, &value); - if (value.length() > max_length) max_length = value.length(); + if (value.length() > max_length) { + max_length = value.length(); + } if (!s.ok() && !s.IsNotFound()) { fprintf(stderr, "get error: %s\n", s.ToString().c_str()); @@ -7640,10 +7739,16 @@ class Benchmark { } bool binary_search(std::vector& data, int start, int end, int key) { - if (data.empty()) return false; - if (start > end) return false; + if (data.empty()) { + return false; + } + if (start > end) { + return false; + } int mid = start + (end - start) / 2; - if (mid > static_cast(data.size()) - 1) return false; + if (mid > static_cast(data.size()) - 1) { + return false; + } if (data[mid] == key) { return true; } else if (data[mid] > key) { @@ -7716,7 +7821,9 @@ class Benchmark { found = binary_search(data, 0, static_cast(data.size() - 1), lookup_key); data.clear(); - if (found) break; + if (found) { + break; + } } std::cout << "Found key? " << std::to_string(found) << "\n"; sp = FLAGS_env->NowNanos(); @@ -7726,7 +7833,9 @@ class Benchmark { std::cout << "Sample data from GetMergeOperands API call: "; for (PinnableSlice& psl : a_slice) { std::cout << "List: " << to_print << " : " << *psl.GetSelf() << "\n"; - if (to_print++ > 2) break; + if (to_print++ > 2) { + break; + } } } @@ -7738,6 +7847,7 @@ class Benchmark { ro.rate_limiter_priority = FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; ro.readahead_size = FLAGS_readahead_size; + ro.auto_readahead_size = FLAGS_auto_readahead_size; Status s = db->VerifyChecksum(ro); if (!s.ok()) { fprintf(stderr, "VerifyChecksum() failed: %s\n", s.ToString().c_str()); @@ -7753,6 +7863,7 @@ class Benchmark { ro.rate_limiter_priority = FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; ro.readahead_size = FLAGS_readahead_size; + ro.auto_readahead_size = FLAGS_auto_readahead_size; Status s = db->VerifyFileChecksums(ro); if (!s.ok()) { fprintf(stderr, "VerifyFileChecksums() failed: %s\n", @@ -8103,58 +8214,24 @@ class Benchmark { } void WaitForCompactionHelper(DBWithColumnFamilies& db) { - // This is an imperfect way of waiting for compaction. The loop and sleep - // is done because a thread that finishes a compaction job should get a - // chance to pickup a new compaction job. - - std::vector keys = {DB::Properties::kMemTableFlushPending, - DB::Properties::kNumRunningFlushes, - DB::Properties::kCompactionPending, - DB::Properties::kNumRunningCompactions}; - fprintf(stdout, "waitforcompaction(%s): started\n", db.db->GetName().c_str()); - while (true) { - bool retry = false; - - for (const auto& k : keys) { - uint64_t v; - if (!db.db->GetIntProperty(k, &v)) { - fprintf(stderr, "waitforcompaction(%s): GetIntProperty(%s) failed\n", - db.db->GetName().c_str(), k.c_str()); - exit(1); - } else if (v > 0) { - fprintf(stdout, - "waitforcompaction(%s): active(%s). Sleep 10 seconds\n", - db.db->GetName().c_str(), k.c_str()); - FLAGS_env->SleepForMicroseconds(10 * 1000000); - retry = true; - break; - } - } + Status s = db.db->WaitForCompact(WaitForCompactOptions()); - if (!retry) { - fprintf(stdout, "waitforcompaction(%s): finished\n", - db.db->GetName().c_str()); - return; - } - } + fprintf(stdout, "waitforcompaction(%s): finished with status (%s)\n", + db.db->GetName().c_str(), s.ToString().c_str()); } void WaitForCompaction() { // Give background threads a chance to wake FLAGS_env->SleepForMicroseconds(5 * 1000000); - // I am skeptical that this check race free. I hope that checking twice - // reduces the chance. if (db_.db != nullptr) { WaitForCompactionHelper(db_); - WaitForCompactionHelper(db_); } else { for (auto& db_with_cfh : multi_dbs_) { WaitForCompactionHelper(db_with_cfh); - WaitForCompactionHelper(db_with_cfh); } } } @@ -8172,7 +8249,9 @@ class Benchmark { real_from_level = std::numeric_limits::max(); for (auto& f : files) { - if (f.level > 0 && f.level < real_from_level) real_from_level = f.level; + if (f.level > 0 && f.level < real_from_level) { + real_from_level = f.level; + } } if (real_from_level == std::numeric_limits::max()) { @@ -8188,10 +8267,11 @@ class Benchmark { std::vector files_to_compact; for (auto& f : files) { - if (f.level == real_from_level) + if (f.level == real_from_level) { files_to_compact.push_back(f.name); - else if (f.level > real_from_level && f.level < next_level) + } else if (f.level > real_from_level && f.level < next_level) { next_level = f.level; + } } if (files_to_compact.empty()) { @@ -8232,10 +8312,14 @@ class Benchmark { void CompactLevel(int from_level) { if (db_.db != nullptr) { - while (!CompactLevelHelper(db_, from_level)) WaitForCompaction(); + while (!CompactLevelHelper(db_, from_level)) { + WaitForCompaction(); + } } for (auto& db_with_cfh : multi_dbs_) { - while (!CompactLevelHelper(db_with_cfh, from_level)) WaitForCompaction(); + while (!CompactLevelHelper(db_with_cfh, from_level)) { + WaitForCompaction(); + } } } @@ -8317,6 +8401,11 @@ class Benchmark { } } + void CacheReportProblems() { + auto debug_logger = std::make_shared(DEBUG_LEVEL); + cache_->ReportProblems(debug_logger); + } + void PrintStats(const char* key) { if (db_.db != nullptr) { PrintStats(db_.db, key, false); @@ -8564,15 +8653,15 @@ int db_bench_tool(int argc, char** argv) { exit(1); } - if (!strcasecmp(FLAGS_compaction_fadvice.c_str(), "NONE")) + if (!strcasecmp(FLAGS_compaction_fadvice.c_str(), "NONE")) { FLAGS_compaction_fadvice_e = ROCKSDB_NAMESPACE::Options::NONE; - else if (!strcasecmp(FLAGS_compaction_fadvice.c_str(), "NORMAL")) + } else if (!strcasecmp(FLAGS_compaction_fadvice.c_str(), "NORMAL")) { FLAGS_compaction_fadvice_e = ROCKSDB_NAMESPACE::Options::NORMAL; - else if (!strcasecmp(FLAGS_compaction_fadvice.c_str(), "SEQUENTIAL")) + } else if (!strcasecmp(FLAGS_compaction_fadvice.c_str(), "SEQUENTIAL")) { FLAGS_compaction_fadvice_e = ROCKSDB_NAMESPACE::Options::SEQUENTIAL; - else if (!strcasecmp(FLAGS_compaction_fadvice.c_str(), "WILLNEED")) + } else if (!strcasecmp(FLAGS_compaction_fadvice.c_str(), "WILLNEED")) { FLAGS_compaction_fadvice_e = ROCKSDB_NAMESPACE::Options::WILLNEED; - else { + } else { fprintf(stdout, "Unknown compaction fadvice:%s\n", FLAGS_compaction_fadvice.c_str()); exit(1); diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 1a3821aa1..93f1f2427 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -31,10 +31,10 @@ default_params = { - "acquire_snapshot_one_in": 10000, + "acquire_snapshot_one_in": lambda: random.choice([100, 10000]), "backup_max_size": 100 * 1024 * 1024, # Consider larger number when backups considered more stable - "backup_one_in": 100000, + "backup_one_in": lambda: random.choice([1000, 100000]), "batch_protection_bytes_per_key": lambda: random.choice([0, 8]), "memtable_protection_bytes_per_key": lambda: random.choice([0, 1, 2, 4, 8]), "block_protection_bytes_per_key": lambda: random.choice([0, 1, 2, 4, 8]), @@ -43,12 +43,12 @@ [random.randint(0, 19), random.lognormvariate(2.3, 1.3)] ), "cache_index_and_filter_blocks": lambda: random.randint(0, 1), - "cache_size": 8388608, + "cache_size": lambda: random.choice([8388608, 33554432]), "charge_compression_dictionary_building_buffer": lambda: random.choice([0, 1]), "charge_filter_construction": lambda: random.choice([0, 1]), "charge_table_reader": lambda: random.choice([0, 1]), "charge_file_metadata": lambda: random.choice([0, 1]), - "checkpoint_one_in": 1000000, + "checkpoint_one_in": lambda: random.choice([10000, 1000000]), "compression_type": lambda: random.choice( ["none", "snappy", "zlib", "lz4", "lz4hc", "xpress", "zstd"] ), @@ -65,9 +65,10 @@ "compression_parallel_threads": 1, "compression_max_dict_buffer_bytes": lambda: (1 << random.randint(0, 40)) - 1, "compression_use_zstd_dict_trainer": lambda: random.randint(0, 1), + "compression_checksum": lambda: random.randint(0, 1), "clear_column_family_one_in": 0, - "compact_files_one_in": 1000000, - "compact_range_one_in": 1000000, + "compact_files_one_in": lambda: random.choice([1000, 1000000]), + "compact_range_one_in": lambda: random.choice([1000, 1000000]), "compaction_pri": random.randint(0, 4), "data_block_index_type": lambda: random.choice([0, 1]), "delpercent": 4, @@ -77,19 +78,19 @@ "enable_compaction_filter": lambda: random.choice([0, 0, 0, 1]), "expected_values_dir": lambda: setup_expected_values_dir(), "fail_if_options_file_error": lambda: random.randint(0, 1), - "flush_one_in": 1000000, - "manual_wal_flush_one_in": lambda: random.choice([0, 0, 1000, 1000000]), + "flush_one_in": lambda: random.choice([1000, 1000000]), + "manual_wal_flush_one_in": lambda: random.choice([0, 1000]), "file_checksum_impl": lambda: random.choice(["none", "crc32c", "xxh64", "big"]), - "get_live_files_one_in": 1000000, + "get_live_files_one_in": lambda: random.choice([10000, 1000000]), # Note: the following two are intentionally disabled as the corresponding # APIs are not guaranteed to succeed. "get_sorted_wal_files_one_in": 0, "get_current_wal_file_one_in": 0, # Temporarily disable hash index "index_type": lambda: random.choice([0, 0, 0, 2, 2, 3]), - "ingest_external_file_one_in": 1000000, + "ingest_external_file_one_in": lambda: random.choice([1000, 1000000]), "iterpercent": 10, - "lock_wal_one_in": 1000000, + "lock_wal_one_in": lambda: random.choice([10000, 1000000]), "mark_for_compaction_one_file_in": lambda: 10 * random.randint(0, 1), "max_background_compactions": 20, "max_bytes_for_level_base": 10485760, @@ -104,7 +105,7 @@ "optimize_filters_for_memory": lambda: random.randint(0, 1), "partition_filters": lambda: random.randint(0, 1), "partition_pinning": lambda: random.randint(0, 3), - "pause_background_one_in": 1000000, + "pause_background_one_in": lambda: random.choice([10000, 1000000]), "prefix_size": lambda: random.choice([-1, 1, 5, 7, 8]), "prefixpercent": 5, "progress_reports": 0, @@ -123,18 +124,24 @@ "use_direct_reads": lambda: random.randint(0, 1), "use_direct_io_for_flush_and_compaction": lambda: random.randint(0, 1), "mock_direct_io": False, - "cache_type": lambda: random.choice(["lru_cache", "hyper_clock_cache"]), + "cache_type": lambda: random.choice( + ["lru_cache", "fixed_hyper_clock_cache", "auto_hyper_clock_cache", + "auto_hyper_clock_cache", "tiered_lru_cache", + "tiered_fixed_hyper_clock_cache", "tiered_auto_hyper_clock_cache", + "tiered_auto_hyper_clock_cache"] + ), "use_full_merge_v1": lambda: random.randint(0, 1), "use_merge": lambda: random.randint(0, 1), # use_put_entity_one_in has to be the same across invocations for verification to work, hence no lambda "use_put_entity_one_in": random.choice([0] * 7 + [1, 5, 10]), # 999 -> use Bloom API - "ribbon_starting_level": lambda: random.choice([random.randint(-1, 10), 999]), + "bloom_before_level": lambda: random.choice([random.randint(-1, 2), random.randint(-1, 10), 0x7fffffff - 1, 0x7fffffff]), "value_size_mult": 32, + "verification_only": 0, "verify_checksum": 1, "write_buffer_size": 4 * 1024 * 1024, "writepercent": 35, - "format_version": lambda: random.choice([2, 3, 4, 5, 5]), + "format_version": lambda: random.choice([2, 3, 4, 5, 6, 6]), "index_block_restart_interval": lambda: random.choice(range(1, 16)), "use_multiget": lambda: random.randint(0, 1), "use_get_entity": lambda: random.choice([0] * 7 + [1]), @@ -153,12 +160,12 @@ "sync": lambda: random.choice([1 if t == 0 else 0 for t in range(0, 20)]), "bytes_per_sync": lambda: random.choice([0, 262144]), "wal_bytes_per_sync": lambda: random.choice([0, 524288]), - # Disable compaction_readahead_size because the test is not passing. - # "compaction_readahead_size" : lambda : random.choice( - # [0, 0, 1024 * 1024]), + "compaction_readahead_size": lambda: random.choice( + [0, 0, 1024 * 1024]), "db_write_buffer_size": lambda: random.choice( [0, 0, 0, 1024 * 1024, 8 * 1024 * 1024, 128 * 1024 * 1024] ), + "use_write_buffer_manager": lambda: random.randint(0, 1), "avoid_unnecessary_blocking_io": random.randint(0, 1), "write_dbid_to_manifest": random.randint(0, 1), "avoid_flush_during_recovery": lambda: random.choice( @@ -168,23 +175,26 @@ [16, 64, 1024 * 1024, 16 * 1024 * 1024] ), "level_compaction_dynamic_level_bytes": lambda: random.randint(0, 1), - "verify_checksum_one_in": 1000000, - "verify_db_one_in": 100000, + "verify_checksum_one_in": lambda: random.choice([100000, 1000000]), + "verify_file_checksums_one_in": lambda: random.choice([100000, 1000000]), + "verify_db_one_in": lambda: random.choice([10000, 100000]), "continuous_verification_interval": 0, "max_key_len": 3, "key_len_percent_dist": "1,30,69", "read_fault_one_in": lambda: random.choice([0, 32, 1000]), + "write_fault_one_in": lambda: random.choice([0, 128, 1000]), "open_metadata_write_fault_one_in": lambda: random.choice([0, 0, 8]), "open_write_fault_one_in": lambda: random.choice([0, 0, 16]), "open_read_fault_one_in": lambda: random.choice([0, 0, 32]), "sync_fault_injection": lambda: random.randint(0, 1), - "get_property_one_in": 1000000, + "get_property_one_in": lambda: random.choice([100000, 1000000]), "paranoid_file_checks": lambda: random.choice([0, 1, 1, 1]), "max_write_buffer_size_to_maintain": lambda: random.choice( [0, 1024 * 1024, 2 * 1024 * 1024, 4 * 1024 * 1024, 8 * 1024 * 1024] ), "user_timestamp_size": 0, "secondary_cache_fault_one_in": lambda: random.choice([0, 0, 32]), + "compressed_secondary_cache_size": lambda: random.choice([8388608, 16777216]), "prepopulate_block_cache": lambda: random.choice([0, 1]), "memtable_prefix_bloom_size_ratio": lambda: random.choice([0.001, 0.01, 0.1, 0.5]), "memtable_whole_key_filtering": lambda: random.randint(0, 1), @@ -196,7 +206,8 @@ "secondary_cache_uri": lambda: random.choice( [ "", - "compressed_secondary_cache://capacity=8388608", + "", + "", "compressed_secondary_cache://capacity=8388608;enable_custom_split_merge=true", ] ), @@ -208,6 +219,13 @@ "num_file_reads_for_auto_readahead": lambda: random.choice([0, 1, 2]), "min_write_buffer_number_to_merge": lambda: random.choice([1, 2]), "preserve_internal_time_seconds": lambda: random.choice([0, 60, 3600, 36000]), + "memtable_max_range_deletions": lambda: random.choice([0] * 6 + [100, 1000]), + # 0 (disable) is the default and more commonly used value. + "bottommost_file_compaction_delay": lambda: random.choice( + [0, 0, 0, 600, 3600, 86400] + ), + "auto_readahead_size" : lambda: random.choice([0, 1]), + "verify_iterator_with_expected_state_one_in": 5, } _TEST_DIR_ENV_VAR = "TEST_TMPDIR" @@ -318,10 +336,14 @@ def is_direct_io_supported(dbname): } whitebox_default_params = { - # TODO: enable this once we figure out how to adjust kill odds for WAL- - # disabled runs, and either (1) separate full `db_stress` runs out of - # whitebox crash or (2) support verification at end of `db_stress` runs - # that ran with WAL disabled. + # TODO: enable this at random once we figure out two things. First, we need + # to ensure the kill odds in WAL-disabled runs result in regular crashing + # before the fifteen minute timeout. When WAL is disabled there are very few + # calls to write functions since writes to SST files are buffered and other + # writes (e.g., MANIFEST) are infrequent. Crashing in reasonable time might + # currently assume killpoints in write functions are reached frequently. + # + # Second, we need to make sure disabling WAL works with `-reopen > 0`. "disable_wal": 0, "duration": 10000, "log2_keys_per_lock": 10, @@ -344,7 +366,6 @@ def is_direct_io_supported(dbname): "write_buffer_size": 32 * 1024 * 1024, "level_compaction_dynamic_level_bytes": lambda: random.randint(0, 1), "paranoid_file_checks": lambda: random.choice([0, 1, 1, 1]), - "verify_iterator_with_expected_state_one_in": 5, # this locks a range of keys } blackbox_simple_default_params = { @@ -367,6 +388,8 @@ def is_direct_io_supported(dbname): "enable_compaction_filter": 0, # `CfConsistencyStressTest::TestIngestExternalFile()` is not implemented. "ingest_external_file_one_in": 0, + # `CfConsistencyStressTest::TestIterateAgainstExpected()` is not implemented. + "verify_iterator_with_expected_state_one_in": 0, } # For pessimistic transaction db @@ -404,6 +427,8 @@ def is_direct_io_supported(dbname): "atomic_flush": 0, "disable_wal": 1, "column_families": 1, + "skip_verifydb": 1, + "verify_db_one_in": 0 } blob_params = { @@ -433,6 +458,8 @@ def is_direct_io_supported(dbname): "test_cf_consistency": 0, "test_batches_snapshots": 0, "user_timestamp_size": 8, + # Below flag is randomly picked once and kept consistent in following runs. + "persist_user_defined_timestamps": random.choice([0, 1, 1]), "use_merge": 0, "use_full_merge_v1": 0, "use_txn": 0, @@ -493,10 +520,15 @@ def is_direct_io_supported(dbname): "enable_compaction_filter": 0, "create_timestamped_snapshot_one_in": 50, "sync_fault_injection": 0, + # This test has aggressive flush frequency and small write buffer size. + # Disabling write fault to avoid writes being stopped. + "write_fault_one_in": 0, # PutEntity in transactions is not yet implemented "use_put_entity_one_in": 0, "use_get_entity": 0, "use_multi_get_entity": 0, + # `MultiOpsTxnsStressTest::TestIterateAgainstExpected()` is not implemented. + "verify_iterator_with_expected_state_one_in": 0, } multiops_wc_txn_params = { @@ -521,7 +553,6 @@ def is_direct_io_supported(dbname): "create_timestamped_snapshot_one_in": 0, } - def finalize_and_sanitize(src_params): dest_params = {k: v() if callable(v) else v for (k, v) in src_params.items()} if is_release_mode(): @@ -642,6 +673,8 @@ def finalize_and_sanitize(src_params): dest_params["enable_compaction_filter"] = 0 dest_params["sync"] = 0 dest_params["write_fault_one_in"] = 0 + dest_params["skip_verifydb"] = 1 + dest_params["verify_db_one_in"] = 0 # Remove the following once write-prepared/write-unprepared with/without # unordered write supports timestamped snapshots if dest_params.get("create_timestamped_snapshot_one_in", 0) > 0: @@ -652,12 +685,56 @@ def finalize_and_sanitize(src_params): if dest_params.get("use_txn") == 1 and dest_params.get("txn_write_policy") != 0: dest_params["sync_fault_injection"] = 0 dest_params["manual_wal_flush_one_in"] = 0 - # PutEntity is currently not supported by SstFileWriter or in conjunction with Merge + # Wide column stress tests require FullMergeV3 if dest_params["use_put_entity_one_in"] != 0: - dest_params["ingest_external_file_one_in"] = 0 - dest_params["use_merge"] = 0 dest_params["use_full_merge_v1"] = 0 - + if dest_params["file_checksum_impl"] == "none": + dest_params["verify_file_checksums_one_in"] = 0 + if dest_params["write_fault_one_in"] > 0: + # background work may be disabled while DB is resuming after some error + dest_params["max_write_buffer_number"] = max(dest_params["max_write_buffer_number"], 10) + if dest_params["secondary_cache_uri"].find("compressed_secondary_cache") >= 0: + dest_params["compressed_secondary_cache_size"] = 0 + dest_params["compressed_secondary_cache_ratio"] = 0.0 + if dest_params["cache_type"].find("tiered_") >= 0: + if dest_params["compressed_secondary_cache_size"] > 0: + dest_params["compressed_secondary_cache_ratio"] = \ + float(dest_params["compressed_secondary_cache_size"]/ \ + (dest_params["cache_size"] + dest_params["compressed_secondary_cache_size"])) + dest_params["compressed_secondary_cache_size"] = 0 + else: + dest_params["compressed_secondary_cache_ratio"] = 0.0 + dest_params["cache_type"] = dest_params["cache_type"].replace("tiered_", "") + else: + if dest_params["secondary_cache_uri"]: + dest_params["compressed_secondary_cache_size"] = 0 + dest_params["compressed_secondary_cache_ratio"] = 0.0 + if dest_params["use_write_buffer_manager"]: + if (dest_params["cache_size"] <= 0 + or dest_params["db_write_buffer_size"] <= 0): + dest_params["use_write_buffer_manager"] = 0 + if dest_params["user_timestamp_size"] > 0 and dest_params["persist_user_defined_timestamps"] == 0: + # Features that are not compatible with UDT in memtable only feature. + dest_params["delpercent"] += dest_params["delrangepercent"] + dest_params["delrangepercent"] = 0 + dest_params["enable_blob_files"] = 0 + dest_params["atomic_flush"] = 0 + dest_params["allow_concurrent_memtable_write"] = 0 + dest_params["block_protection_bytes_per_key"] = 0 + # TODO(yuzhangyu): make stress test logic handle this and enable testing + # these APIs. + # These operations need to compare side to side one operation with another. + # It's hard to guarantee their consistency because when timestamps can be + # collapsed, only operations using the same SuperVersion can be consistent + # with each other. There is no external APIs to ensure that. + dest_params["use_multiget"] = 0 + dest_params["use_multi_get_entity"] = 0 + dest_params["readpercent"] += dest_params.get("iterpercent", 10); + dest_params["iterpercent"] = 0 + # Only best efforts recovery test support disabling wal and + # disable atomic flush. + if dest_params["test_best_efforts_recovery"] == 0: + dest_params["disable_wal"] = 0 return dest_params @@ -734,7 +811,7 @@ def gen_cmd(params, unknown_params): "stress_cmd", "test_tiered_storage", "cleanup_cmd", - "skip_tmpdir_check" + "skip_tmpdir_check", } and v is not None ] @@ -743,7 +820,7 @@ def gen_cmd(params, unknown_params): return cmd -def execute_cmd(cmd, timeout): +def execute_cmd(cmd, timeout=None): child = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) print("Running db_stress with pid=%d: %s\n\n" % (child.pid, " ".join(cmd))) @@ -760,6 +837,24 @@ def execute_cmd(cmd, timeout): return hit_timeout, child.returncode, outs.decode("utf-8"), errs.decode("utf-8") +def exit_if_stderr_has_errors(stderr, print_stderr=True): + if print_stderr: + for line in stderr.split("\n"): + if line != "" and not line.startswith("WARNING"): + print("stderr has error message:") + print("***" + line + "***") + + stderrdata = stderr.lower() + errorcount = stderrdata.count("error") - stderrdata.count("got errors 0 times") + print("#times error occurred in output is " + str(errorcount) + "\n") + + if errorcount > 0: + print("TEST FAILED. Output has 'error'!!!\n") + sys.exit(2) + if stderrdata.find("fail") >= 0: + print("TEST FAILED. Output has 'fail'!!!\n") + sys.exit(2) + # This script runs and kills db_stress multiple times. It checks consistency # in case of unsafe crashes in RocksDB. def blackbox_crash_main(args, unknown_args): @@ -792,15 +887,28 @@ def blackbox_crash_main(args, unknown_args): print(errs) sys.exit(2) - for line in errs.split("\n"): - if line != "" and not line.startswith("WARNING"): - print("stderr has error message:") - print("***" + line + "***") + exit_if_stderr_has_errors(errs); time.sleep(1) # time to stabilize before the next run time.sleep(1) # time to stabilize before the next run + # We should run the test one more time with VerifyOnly setup and no-timeout + # Only do this if the tests are not failed for total-duration + print("Running final time for verification") + cmd_params.update({"verification_only": 1}) + cmd_params.update({"skip_verifydb": 0}) + + cmd = gen_cmd( + dict(list(cmd_params.items()) + list({"db": dbname}.items())), unknown_args + ) + hit_timeout, retcode, outs, errs = execute_cmd(cmd) + + # Print stats of the final run + print("stdout:", outs) + + exit_if_stderr_has_errors(errs) + # we need to clean up after ourselves -- only do this on test success shutil.rmtree(dbname, True) @@ -961,16 +1069,8 @@ def whitebox_crash_main(args, unknown_args): print("TEST FAILED. See kill option and exit code above!!!\n") sys.exit(1) - stderrdata = stderrdata.lower() - errorcount = stderrdata.count("error") - stderrdata.count("got errors 0 times") - print("#times error occurred in output is " + str(errorcount) + "\n") - - if errorcount > 0: - print("TEST FAILED. Output has 'error'!!!\n") - sys.exit(2) - if stderrdata.find("fail") >= 0: - print("TEST FAILED. Output has 'fail'!!!\n") - sys.exit(2) + #stderr already printed above + exit_if_stderr_has_errors(stderrdata, False) # First half of the duration, keep doing kill test. For the next half, # try different modes. diff --git a/tools/dump/db_dump_tool.cc b/tools/dump/db_dump_tool.cc index 535e70c43..e234c9a73 100644 --- a/tools/dump/db_dump_tool.cc +++ b/tools/dump/db_dump_tool.cc @@ -195,16 +195,20 @@ bool DbUndumpTool::Run(const UndumpOptions& undump_options, std::unique_ptr keyscratch(new char[last_keysize]); std::unique_ptr valscratch(new char[last_valsize]); - while (1) { + while (true) { uint32_t keysize, valsize; ROCKSDB_NAMESPACE::Slice keyslice; ROCKSDB_NAMESPACE::Slice valslice; status = dumpfile->Read(4, &slice, scratch8); - if (!status.ok() || slice.size() != 4) break; + if (!status.ok() || slice.size() != 4) { + break; + } keysize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data()); if (keysize > last_keysize) { - while (keysize > last_keysize) last_keysize *= 2; + while (keysize > last_keysize) { + last_keysize *= 2; + } keyscratch = std::unique_ptr(new char[last_keysize]); } @@ -225,7 +229,9 @@ bool DbUndumpTool::Run(const UndumpOptions& undump_options, } valsize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data()); if (valsize > last_valsize) { - while (valsize > last_valsize) last_valsize *= 2; + while (valsize > last_valsize) { + last_valsize *= 2; + } valscratch = std::unique_ptr(new char[last_valsize]); } diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index 46dd36ccc..1e7feb712 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -6,7 +6,7 @@ // #include "rocksdb/utilities/ldb_cmd.h" -#include +#include #include #include #include @@ -22,6 +22,8 @@ #include "db/dbformat.h" #include "db/log_reader.h" #include "db/version_util.h" +#include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" #include "db/write_batch_internal.h" #include "file/filename.h" #include "rocksdb/cache.h" @@ -129,7 +131,7 @@ LDBCommand* LDBCommand::InitFromCmdLineArgs( const std::vector* column_families) { std::vector args; for (int i = 1; i < argc; i++) { - args.push_back(argv[i]); + args.emplace_back(argv[i]); } return InitFromCmdLineArgs(args, options, ldb_options, column_families, SelectCommand); @@ -206,9 +208,15 @@ LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) { if (parsed_params.cmd == GetCommand::Name()) { return new GetCommand(parsed_params.cmd_params, parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == GetEntityCommand::Name()) { + return new GetEntityCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); } else if (parsed_params.cmd == PutCommand::Name()) { return new PutCommand(parsed_params.cmd_params, parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == PutEntityCommand::Name()) { + return new PutEntityCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); } else if (parsed_params.cmd == BatchPutCommand::Name()) { return new BatchPutCommand(parsed_params.cmd_params, parsed_params.option_map, parsed_params.flags); @@ -924,7 +932,15 @@ void LDBCommand::PrepareOptions() { &column_families_); if (!s.ok() && !s.IsNotFound()) { // Option file exists but load option file error. - std::string msg = s.ToString(); + std::string current_version = std::to_string(ROCKSDB_MAJOR) + "." + + std::to_string(ROCKSDB_MINOR) + "." + + std::to_string(ROCKSDB_PATCH); + std::string msg = + s.ToString() + "\nThis tool was built with version " + + current_version + + ". If your db is in a different version, please try again " + "with option --" + + LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS + "."; exec_state_ = LDBCommandExecuteResult::Failed(msg); db_ = nullptr; return; @@ -968,7 +984,7 @@ void LDBCommand::PrepareOptions() { // existing DB. if (st.ok() && cf_list.size() > 1) { // Ignore single column family DB. - for (auto cf_name : cf_list) { + for (const auto& cf_name : cf_list) { column_families_.emplace_back(cf_name, options_); } } @@ -1080,6 +1096,28 @@ std::string LDBCommand::PrintKeyValue(const std::string& key, return PrintKeyValue(key, value, is_hex, is_hex); } +std::string LDBCommand::PrintKeyValueOrWideColumns( + const Slice& key, const Slice& value, const WideColumns& wide_columns, + bool is_key_hex, bool is_value_hex) { + if (wide_columns.empty() || + WideColumnsHelper::HasDefaultColumnOnly(wide_columns)) { + return PrintKeyValue(key.ToString(), value.ToString(), is_key_hex, + is_value_hex); + } + /* + // Sample plaintext output (first column is kDefaultWideColumnName) + key_1 ==> :foo attr_name1:bar attr_name2:baz + + // Sample hex output (first column is kDefaultWideColumnName) + 0x6669727374 ==> :0x68656C6C6F 0x617474725F6E616D6531:0x666F6F + */ + std::ostringstream oss; + WideColumnsHelper::DumpWideColumns(wide_columns, oss, is_value_hex); + return PrintKeyValue(key.ToString(), oss.str().c_str(), is_key_hex, + false); // is_value_hex_ is already honored in oss. + // avoid double-hexing it. +} + std::string LDBCommand::HelpRangeCmdArgs() { std::ostringstream str_stream; str_stream << " "; @@ -1326,7 +1364,9 @@ void DumpManifestFile(Options options, std::string file, bool verbose, bool hex, ImmutableDBOptions immutable_db_options(options); VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc, /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ ""); + /*db_id=*/"", /*db_session_id=*/"", + options.daily_offpeak_time_utc, + /*error_handler=*/nullptr); Status s = versions.DumpManifest(options, file, verbose, hex, json, cf_descs); if (!s.ok()) { fprintf(stderr, "Error in processing file %s %s\n", file.c_str(), @@ -1357,8 +1397,7 @@ ManifestDumpCommand::ManifestDumpCommand( options, flags, false, BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})), verbose_(false), - json_(false), - path_("") { + json_(false) { verbose_ = IsFlagPresent(flags, ARG_VERBOSE); json_ = IsFlagPresent(flags, ARG_JSON); @@ -1372,6 +1411,7 @@ ManifestDumpCommand::ManifestDumpCommand( } void ManifestDumpCommand::DoCommand() { + PrepareOptions(); std::string manifestfile; if (!path_.empty()) { @@ -1468,7 +1508,9 @@ Status GetLiveFilesChecksumInfoFromVersionSet(Options options, ImmutableDBOptions immutable_db_options(options); VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc, /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ ""); + /*db_id=*/"", /*db_session_id=*/"", + options.daily_offpeak_time_utc, + /*error_handler=*/nullptr); std::vector cf_name_list; s = versions.ListColumnFamilies(&cf_name_list, db_path, immutable_db_options.fs.get()); @@ -1501,8 +1543,7 @@ FileChecksumDumpCommand::FileChecksumDumpCommand( const std::map& options, const std::vector& flags) : LDBCommand(options, flags, false, - BuildCmdLineOptions({ARG_PATH, ARG_HEX})), - path_("") { + BuildCmdLineOptions({ARG_PATH, ARG_HEX})) { auto itr = options.find(ARG_PATH); if (itr != options.end()) { path_ = itr->second; @@ -1514,6 +1555,7 @@ FileChecksumDumpCommand::FileChecksumDumpCommand( } void FileChecksumDumpCommand::DoCommand() { + PrepareOptions(); // print out the checksum information in the following format: // sst file number, checksum function name, checksum value // sst file number, checksum function name, checksum value @@ -1618,6 +1660,7 @@ ListColumnFamiliesCommand::ListColumnFamiliesCommand( : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {} void ListColumnFamiliesCommand::DoCommand() { + PrepareOptions(); std::vector column_families; Status s = DB::ListColumnFamilies(options_, db_path_, &column_families); if (!s.ok()) { @@ -1626,7 +1669,7 @@ void ListColumnFamiliesCommand::DoCommand() { } else { fprintf(stdout, "Column families in %s: \n{", db_path_.c_str()); bool first = true; - for (auto cf : column_families) { + for (const auto& cf : column_families) { if (!first) { fprintf(stdout, ", "); } @@ -1859,11 +1902,16 @@ void InternalDumpCommand::DoCommand() { s1 = 0; row = ikey.Encode().ToString(); val = key_version.value; - for (k = 0; row[k] != '\x01' && row[k] != '\0'; k++) s1++; - for (k = 0; val[k] != '\x01' && val[k] != '\0'; k++) s1++; + for (k = 0; row[k] != '\x01' && row[k] != '\0'; k++) { + s1++; + } + for (k = 0; val[k] != '\x01' && val[k] != '\0'; k++) { + s1++; + } for (int j = 0; row[j] != delim_[0] && row[j] != '\0' && row[j] != '\x01'; - j++) + j++) { rtype1 += row[j]; + } if (rtype2.compare("") && rtype2.compare(rtype1) != 0) { fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n", rtype2.c_str(), c, s2); @@ -1881,8 +1929,20 @@ void InternalDumpCommand::DoCommand() { std::string key = ikey.DebugString(is_key_hex_); Slice value(key_version.value); if (!decode_blob_index_ || value_type != kTypeBlobIndex) { - fprintf(stdout, "%s => %s\n", key.c_str(), - value.ToString(is_value_hex_).c_str()); + if (value_type == kTypeWideColumnEntity) { + std::ostringstream oss; + const Status s = WideColumnsHelper::DumpSliceAsWideColumns( + value, oss, is_value_hex_); + if (!s.ok()) { + fprintf(stderr, "%s => error deserializing wide columns\n", + key.c_str()); + } else { + fprintf(stdout, "%s => %s\n", key.c_str(), oss.str().c_str()); + } + } else { + fprintf(stdout, "%s => %s\n", key.c_str(), + value.ToString(is_value_hex_).c_str()); + } } else { BlobIndex blob_index; @@ -1897,7 +1957,9 @@ void InternalDumpCommand::DoCommand() { } // Terminate if maximum number of keys have been dumped - if (max_keys_ > 0 && count >= max_keys_) break; + if (max_keys_ > 0 && count >= max_keys_) { + break; + } } if (count_delim_) { fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n", @@ -2136,9 +2198,13 @@ void DBDumperCommand::DoDumpCommand() { for (; iter->Valid(); iter->Next()) { int rawtime = 0; // If end marker was specified, we stop before it - if (!null_to_ && (iter->key().ToString() >= to_)) break; + if (!null_to_ && (iter->key().ToString() >= to_)) { + break; + } // Terminate if maximum number of keys have been dumped - if (max_keys == 0) break; + if (max_keys == 0) { + break; + } if (is_db_ttl_) { TtlIterator* it_ttl = static_cast_with_check(iter); rawtime = it_ttl->ttl_timestamp(); @@ -2159,8 +2225,9 @@ void DBDumperCommand::DoDumpCommand() { row = iter->key().ToString(); val = iter->value().ToString(); s1 = row.size() + val.size(); - for (int j = 0; row[j] != delim_[0] && row[j] != '\0'; j++) + for (int j = 0; row[j] != delim_[0] && row[j] != '\0'; j++) { rtype1 += row[j]; + } if (rtype2.compare("") && rtype2.compare(rtype1) != 0) { fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n", rtype2.c_str(), c, s2); @@ -2182,9 +2249,14 @@ void DBDumperCommand::DoDumpCommand() { if (is_db_ttl_ && timestamp_) { fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str()); } + // (TODO) TTL Iterator does not support wide columns yet. std::string str = - PrintKeyValue(iter->key().ToString(), iter->value().ToString(), - is_key_hex_, is_value_hex_); + is_db_ttl_ + ? PrintKeyValue(iter->key().ToString(), iter->value().ToString(), + is_key_hex_, is_value_hex_) + : PrintKeyValueOrWideColumns(iter->key(), iter->value(), + iter->columns(), is_key_hex_, + is_value_hex_); fprintf(stdout, "%s\n", str.c_str()); } } @@ -2232,7 +2304,7 @@ ReduceDBLevelsCommand::ReduceDBLevelsCommand( std::vector ReduceDBLevelsCommand::PrepareArgs( const std::string& db_path, int new_levels, bool print_old_level) { std::vector ret; - ret.push_back("reduce_levels"); + ret.emplace_back("reduce_levels"); ret.push_back("--" + ARG_DB + "=" + db_path); ret.push_back("--" + ARG_NEW_LEVELS + "=" + std::to_string(new_levels)); if (print_old_level) { @@ -2270,7 +2342,9 @@ Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt, int* levels) { WriteBufferManager wb(opt.db_write_buffer_size); VersionSet versions(db_path_, &db_options, soptions, tc.get(), &wb, &wc, /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, - /*db_id*/ "", /*db_session_id*/ ""); + /*db_id=*/"", /*db_session_id=*/"", + opt.daily_offpeak_time_utc, + /*error_handler=*/nullptr); std::vector dummy; ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName, ColumnFamilyOptions(opt)); @@ -2526,6 +2600,16 @@ class InMemoryHandler : public WriteBatch::Handler { return Status::OK(); } + Status PutEntityCF(uint32_t cf, const Slice& key, + const Slice& value) override { + row_ << "PUT_ENTITY(" << cf << ") : "; + std::string k = LDBCommand::StringToHex(key.ToString()); + if (print_values_) { + return WideColumnsHelper::DumpSliceAsWideColumns(value, row_, true); + } + return Status::OK(); + } + Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override { row_ << "MERGE(" << cf << ") : "; commonPutMerge(key, value); @@ -2589,7 +2673,7 @@ class InMemoryHandler : public WriteBatch::Handler { return Status::OK(); } - ~InMemoryHandler() override {} + ~InMemoryHandler() override = default; protected: Handler::OptionState WriteAfterCommit() const override { @@ -2628,8 +2712,9 @@ void DumpWalFile(Options options, std::string wal_file, bool print_header, // we need the log number, but ParseFilename expects dbname/NNN.log. std::string sanitized = wal_file; size_t lastslash = sanitized.rfind('/'); - if (lastslash != std::string::npos) + if (lastslash != std::string::npos) { sanitized = sanitized.substr(lastslash + 1); + } if (!ParseFileName(sanitized, &log_number, &type)) { // bogus input, carry on as best we can log_number = 0; @@ -2731,6 +2816,7 @@ void WALDumperCommand::Help(std::string& ret) { } void WALDumperCommand::DoCommand() { + PrepareOptions(); DumpWalFile(options_, wal_file_, print_header_, print_values_, is_write_committed_, &exec_state_); } @@ -2782,6 +2868,55 @@ void GetCommand::DoCommand() { // ---------------------------------------------------------------------------- +GetEntityCommand::GetEntityCommand( + const std::vector& params, + const std::map& options, + const std::vector& flags) + : LDBCommand( + options, flags, true, + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + if (params.size() != 1) { + exec_state_ = LDBCommandExecuteResult::Failed( + " must be specified for the get_entity command"); + } else { + key_ = params.at(0); + } + + if (is_key_hex_) { + key_ = HexToString(key_); + } +} + +void GetEntityCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(GetEntityCommand::Name()); + ret.append(" "); + ret.append(" [--" + ARG_TTL + "]"); + ret.append("\n"); +} + +void GetEntityCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + PinnableWideColumns pinnable_wide_columns; + Status st = db_->GetEntity(ReadOptions(), GetCfHandle(), key_, + &pinnable_wide_columns); + if (st.ok()) { + std::ostringstream oss; + WideColumnsHelper::DumpWideColumns(pinnable_wide_columns.columns(), oss, + is_value_hex_); + fprintf(stdout, "%s\n", oss.str().c_str()); + } else { + std::stringstream oss; + oss << "GetEntity failed: " << st.ToString(); + exec_state_ = LDBCommandExecuteResult::Failed(oss.str()); + } +} + +// ---------------------------------------------------------------------------- + ApproxSizeCommand::ApproxSizeCommand( const std::vector& /*params*/, const std::map& options, @@ -2855,9 +2990,8 @@ BatchPutCommand::BatchPutCommand( for (size_t i = 0; i < params.size(); i += 2) { std::string key = params.at(i); std::string value = params.at(i + 1); - key_values_.push_back(std::pair( - is_key_hex_ ? HexToString(key) : key, - is_value_hex_ ? HexToString(value) : value)); + key_values_.emplace_back(is_key_hex_ ? HexToString(key) : key, + is_value_hex_ ? HexToString(value) : value); } } create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); @@ -3024,30 +3158,22 @@ void ScanCommand::DoCommand() { } } - Slice key_slice = it->key(); - - std::string formatted_key; - if (is_key_hex_) { - formatted_key = "0x" + key_slice.ToString(true /* hex */); - key_slice = formatted_key; - } else if (ldb_options_.key_formatter) { - formatted_key = ldb_options_.key_formatter->Format(key_slice); - key_slice = formatted_key; - } - if (no_value_) { - fprintf(stdout, "%.*s\n", static_cast(key_slice.size()), - key_slice.data()); - } else { - Slice val_slice = it->value(); - std::string formatted_value; - if (is_value_hex_) { - formatted_value = "0x" + val_slice.ToString(true /* hex */); - val_slice = formatted_value; + std::string key_str = it->key().ToString(); + if (is_key_hex_) { + key_str = StringToHex(key_str); + } else if (ldb_options_.key_formatter) { + key_str = ldb_options_.key_formatter->Format(key_str); } - fprintf(stdout, "%.*s : %.*s\n", static_cast(key_slice.size()), - key_slice.data(), static_cast(val_slice.size()), - val_slice.data()); + fprintf(stdout, "%s\n", key_str.c_str()); + } else { + std::string str = is_db_ttl_ ? PrintKeyValue(it->key().ToString(), + it->value().ToString(), + is_key_hex_, is_value_hex_) + : PrintKeyValueOrWideColumns( + it->key(), it->value(), it->columns(), + is_key_hex_, is_value_hex_); + fprintf(stdout, "%s\n", str.c_str()); } num_keys_scanned++; @@ -3226,6 +3352,81 @@ void PutCommand::OverrideBaseOptions() { // ---------------------------------------------------------------------------- +PutEntityCommand::PutEntityCommand( + const std::vector& params, + const std::map& options, + const std::vector& flags) + : LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, + ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) { + if (params.size() < 2) { + exec_state_ = LDBCommandExecuteResult::Failed( + " and at least one column : must be " + "specified for the put_entity command"); + } else { + auto iter = params.begin(); + key_ = *iter; + if (is_key_hex_) { + key_ = HexToString(key_); + } + for (++iter; iter != params.end(); ++iter) { + auto split = StringSplit(*iter, ':'); + if (split.size() != 2) { + exec_state_ = LDBCommandExecuteResult::Failed( + "wide column format needs to be : (did " + "you mean put ?)"); + return; + } + std::string name(split[0]); + std::string value(split[1]); + if (is_value_hex_) { + name = HexToString(name); + value = HexToString(value); + } + column_names_.push_back(name); + column_values_.push_back(value); + } + } + create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); +} + +void PutEntityCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(PutCommand::Name()); + ret.append( + " : : " + "<...>"); + ret.append(" [--" + ARG_CREATE_IF_MISSING + "]"); + ret.append(" [--" + ARG_TTL + "]"); + ret.append("\n"); +} + +void PutEntityCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + assert(column_names_.size() == column_values_.size()); + WideColumns columns; + for (size_t i = 0; i < column_names_.size(); i++) { + WideColumn column(column_names_[i], column_values_[i]); + columns.emplace_back(column); + } + Status st = db_->PutEntity(WriteOptions(), GetCfHandle(), key_, columns); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } +} + +void PutEntityCommand::OverrideBaseOptions() { + LDBCommand::OverrideBaseOptions(); + options_.create_if_missing = create_if_missing_; +} + +// ---------------------------------------------------------------------------- + const char* DBQuerierCommand::HELP_CMD = "help"; const char* DBQuerierCommand::GET_CMD = "get"; const char* DBQuerierCommand::PUT_CMD = "put"; diff --git a/tools/ldb_cmd_impl.h b/tools/ldb_cmd_impl.h index 97de981b1..2a396754d 100644 --- a/tools/ldb_cmd_impl.h +++ b/tools/ldb_cmd_impl.h @@ -403,6 +403,22 @@ class GetCommand : public LDBCommand { std::string key_; }; +class GetEntityCommand : public LDBCommand { + public: + static std::string Name() { return "get_entity"; } + + GetEntityCommand(const std::vector& params, + const std::map& options, + const std::vector& flags); + + void DoCommand() override; + + static void Help(std::string& ret); + + private: + std::string key_; +}; + class ApproxSizeCommand : public LDBCommand { public: static std::string Name() { return "approxsize"; } @@ -530,6 +546,26 @@ class PutCommand : public LDBCommand { std::string value_; }; +class PutEntityCommand : public LDBCommand { + public: + static std::string Name() { return "put_entity"; } + + PutEntityCommand(const std::vector& params, + const std::map& options, + const std::vector& flags); + + void DoCommand() override; + + static void Help(std::string& ret); + + void OverrideBaseOptions() override; + + private: + std::string key_; + std::vector column_names_; + std::vector column_values_; +}; + /** * Command that starts up a REPL shell that allows * get/put/delete. diff --git a/tools/ldb_cmd_test.cc b/tools/ldb_cmd_test.cc index c5b4115d1..8e43972d4 100644 --- a/tools/ldb_cmd_test.cc +++ b/tools/ldb_cmd_test.cc @@ -185,7 +185,7 @@ class FileChecksumTestHelper { public: FileChecksumTestHelper(Options& options, DB* db, std::string db_name) : options_(options), db_(db), dbname_(db_name) {} - ~FileChecksumTestHelper() {} + ~FileChecksumTestHelper() = default; // Verify the checksum information in Manifest. Status VerifyChecksumInManifest( @@ -207,7 +207,8 @@ class FileChecksumTestHelper { WriteBufferManager wb(options_.db_write_buffer_size); ImmutableDBOptions immutable_db_options(options_); VersionSet versions(dbname_, &immutable_db_options, sopt, tc.get(), &wb, - &wc, nullptr, nullptr, "", ""); + &wc, nullptr, nullptr, "", "", + options_.daily_offpeak_time_utc, nullptr); std::vector cf_name_list; Status s; s = versions.ListColumnFamilies(&cf_name_list, dbname_, @@ -232,8 +233,8 @@ class FileChecksumTestHelper { return Status::Corruption("The number of files does not match!"); } for (size_t i = 0; i < live_files.size(); i++) { - std::string stored_checksum = ""; - std::string stored_func_name = ""; + std::string stored_checksum; + std::string stored_func_name; s = checksum_list->SearchOneFileChecksum( live_files[i].file_number, &stored_checksum, &stored_func_name); if (s.IsNotFound()) { @@ -268,7 +269,7 @@ class FileChecksumTestHelper { break; } } - EXPECT_OK(db_->EnableFileDeletions()); + EXPECT_OK(db_->EnableFileDeletions(/*force=*/false)); return cs; } }; @@ -633,9 +634,9 @@ TEST_F(LdbCmdTest, OptionParsing) { opts.env = TryLoadCustomOrDefaultEnv(); { std::vector args; - args.push_back("scan"); - args.push_back("--ttl"); - args.push_back("--timestamp"); + args.emplace_back("scan"); + args.emplace_back("--ttl"); + args.emplace_back("--timestamp"); LDBCommand* command = ROCKSDB_NAMESPACE::LDBCommand::InitFromCmdLineArgs( args, opts, LDBOptions(), nullptr); const std::vector flags = command->TEST_GetFlags(); @@ -647,9 +648,9 @@ TEST_F(LdbCmdTest, OptionParsing) { // test parsing options which contains equal sign in the option value { std::vector args; - args.push_back("scan"); - args.push_back("--db=/dev/shm/ldbtest/"); - args.push_back( + args.emplace_back("scan"); + args.emplace_back("--db=/dev/shm/ldbtest/"); + args.emplace_back( "--from='abcd/efg/hijk/lmn/" "opq:__rst.uvw.xyz?a=3+4+bcd+efghi&jk=lm_no&pq=rst-0&uv=wx-8&yz=a&bcd_" "ef=gh.ijk'"); diff --git a/tools/ldb_test.py b/tools/ldb_test.py index e243d69c0..cde041471 100644 --- a/tools/ldb_test.py +++ b/tools/ldb_test.py @@ -121,32 +121,55 @@ def testSimpleStringPutGet(self): self.assertRunOK("get x2", "y2") self.assertRunFAIL("get x3") - self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2") + self.assertRunFAIL("put_entity x4") + self.assertRunFAIL("put_entity x4 cv1") + self.assertRunOK("put_entity x4 :cv1", "OK") + self.assertRunOK("get_entity x4", ":cv1") + + self.assertRunOK("put_entity x5 cn1:cv1 cn2:cv2", "OK") + self.assertRunOK("get_entity x5", "cn1:cv1 cn2:cv2") + + self.assertRunOK( + "scan --from=x1 --to=z", + "x1 ==> y1\nx2 ==> y2\nx4 ==> cv1\nx5 ==> cn1:cv1 cn2:cv2", + ) self.assertRunOK("put x3 y3", "OK") - self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2\nx3 : y3") - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3") - self.assertRunOK("scan --from=x", "x1 : y1\nx2 : y2\nx3 : y3") + self.assertRunOK( + "scan --from=x1 --to=z", + "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> cv1\nx5 ==> cn1:cv1 cn2:cv2", + ) + self.assertRunOK( + "scan", + "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> cv1\nx5 ==> cn1:cv1 cn2:cv2", + ) + self.assertRunOK( + "scan --from=x", + "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> cv1\nx5 ==> cn1:cv1 cn2:cv2", + ) - self.assertRunOK("scan --to=x2", "x1 : y1") - self.assertRunOK("scan --from=x1 --to=z --max_keys=1", "x1 : y1") - self.assertRunOK("scan --from=x1 --to=z --max_keys=2", "x1 : y1\nx2 : y2") + self.assertRunOK("scan --to=x2", "x1 ==> y1") + self.assertRunOK("scan --from=x1 --to=z --max_keys=1", "x1 ==> y1") + self.assertRunOK("scan --from=x1 --to=z --max_keys=2", "x1 ==> y1\nx2 ==> y2") + + self.assertRunOK("delete x4", "OK") + self.assertRunOK("delete x5", "OK") self.assertRunOK( - "scan --from=x1 --to=z --max_keys=3", "x1 : y1\nx2 : y2\nx3 : y3" + "scan --from=x1 --to=z --max_keys=3", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3" ) self.assertRunOK( - "scan --from=x1 --to=z --max_keys=4", "x1 : y1\nx2 : y2\nx3 : y3" + "scan --from=x1 --to=z --max_keys=4", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3" ) - self.assertRunOK("scan --from=x1 --to=x2", "x1 : y1") - self.assertRunOK("scan --from=x2 --to=x4", "x2 : y2\nx3 : y3") + self.assertRunOK("scan --from=x1 --to=x2", "x1 ==> y1") + self.assertRunOK("scan --from=x2 --to=x4", "x2 ==> y2\nx3 ==> y3") self.assertRunFAIL("scan --from=x4 --to=z") # No results => FAIL self.assertRunFAIL("scan --from=x1 --to=z --max_keys=foo") - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3") self.assertRunOK("delete x1", "OK") - self.assertRunOK("scan", "x2 : y2\nx3 : y3") + self.assertRunOK("scan", "x2 ==> y2\nx3 ==> y3") self.assertRunOK("delete NonExistentKey", "OK") # It is weird that GET and SCAN raise exception for @@ -171,9 +194,9 @@ def ingestExternSst(self, params, inputSst): def testStringBatchPut(self): print("Running testStringBatchPut...") self.assertRunOK("batchput x1 y1 --create_if_missing", "OK") - self.assertRunOK("scan", "x1 : y1") + self.assertRunOK("scan", "x1 ==> y1") self.assertRunOK('batchput x2 y2 x3 y3 "x4 abc" "y4 xyz"', "OK") - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 abc ==> y4 xyz") self.assertRunFAIL("batchput") self.assertRunFAIL("batchput k1") self.assertRunFAIL("batchput k1 v1 k2") @@ -183,11 +206,11 @@ def testBlobBatchPut(self): dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) self.assertRunOK("batchput x1 y1 --create_if_missing --enable_blob_files", "OK") - self.assertRunOK("scan", "x1 : y1") + self.assertRunOK("scan", "x1 ==> y1") self.assertRunOK( 'batchput --enable_blob_files x2 y2 x3 y3 "x4 abc" "y4 xyz"', "OK" ) - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 abc ==> y4 xyz") blob_files = self.getBlobFiles(dbPath) self.assertTrue(len(blob_files) >= 1) @@ -278,12 +301,12 @@ def testInvalidCmdLines(self): def testHexPutGet(self): print("Running testHexPutGet...") self.assertRunOK("put a1 b1 --create_if_missing", "OK") - self.assertRunOK("scan", "a1 : b1") - self.assertRunOK("scan --hex", "0x6131 : 0x6231") + self.assertRunOK("scan", "a1 ==> b1") + self.assertRunOK("scan --hex", "0x6131 ==> 0x6231") self.assertRunFAIL("put --hex 6132 6232") self.assertRunOK("put --hex 0x6132 0x6232", "OK") - self.assertRunOK("scan --hex", "0x6131 : 0x6231\n0x6132 : 0x6232") - self.assertRunOK("scan", "a1 : b1\na2 : b2") + self.assertRunOK("scan --hex", "0x6131 ==> 0x6231\n0x6132 ==> 0x6232") + self.assertRunOK("scan", "a1 ==> b1\na2 ==> b2") self.assertRunOK("get a1", "b1") self.assertRunOK("get --hex 0x6131", "0x6231") self.assertRunOK("get a2", "b2") @@ -292,27 +315,28 @@ def testHexPutGet(self): self.assertRunOK("get --key_hex --value_hex 0x6132", "0x6232") self.assertRunOK("get --value_hex a2", "0x6232") self.assertRunOK( - "scan --key_hex --value_hex", "0x6131 : 0x6231\n0x6132 : 0x6232" + "scan --key_hex --value_hex", "0x6131 ==> 0x6231\n0x6132 ==> 0x6232" ) self.assertRunOK( - "scan --hex --from=0x6131 --to=0x6133", "0x6131 : 0x6231\n0x6132 : 0x6232" + "scan --hex --from=0x6131 --to=0x6133", + "0x6131 ==> 0x6231\n0x6132 ==> 0x6232", ) - self.assertRunOK("scan --hex --from=0x6131 --to=0x6132", "0x6131 : 0x6231") - self.assertRunOK("scan --key_hex", "0x6131 : b1\n0x6132 : b2") - self.assertRunOK("scan --value_hex", "a1 : 0x6231\na2 : 0x6232") + self.assertRunOK("scan --hex --from=0x6131 --to=0x6132", "0x6131 ==> 0x6231") + self.assertRunOK("scan --key_hex", "0x6131 ==> b1\n0x6132 ==> b2") + self.assertRunOK("scan --value_hex", "a1 ==> 0x6231\na2 ==> 0x6232") self.assertRunOK("batchput --hex 0x6133 0x6233 0x6134 0x6234", "OK") - self.assertRunOK("scan", "a1 : b1\na2 : b2\na3 : b3\na4 : b4") + self.assertRunOK("scan", "a1 ==> b1\na2 ==> b2\na3 ==> b3\na4 ==> b4") self.assertRunOK("delete --hex 0x6133", "OK") - self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4") + self.assertRunOK("scan", "a1 ==> b1\na2 ==> b2\na4 ==> b4") self.assertRunOK("checkconsistency", "OK") def testTtlPutGet(self): print("Running testTtlPutGet...") self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK") - self.assertRunOK("scan --hex", "0x6131 : 0x6231", True) + self.assertRunOK("scan --hex", "0x6131 ==> 0x6231", True) self.assertRunOK("dump --ttl ", "a1 ==> b1", True) self.assertRunOK("dump --hex --ttl ", "0x6131 ==> 0x6231\nKeys in range: 1") - self.assertRunOK("scan --hex --ttl", "0x6131 : 0x6231") + self.assertRunOK("scan --hex --ttl", "0x6131 ==> 0x6231") self.assertRunOK("get --value_hex a1", "0x6231", True) self.assertRunOK("get --ttl a1", "b1") self.assertRunOK("put a3 b3 --create_if_missing", "OK") @@ -334,7 +358,7 @@ def testInvalidCmdLines(self): # noqa: F811 T25377293 Grandfathered in def testDumpLoad(self): print("Running testDumpLoad...") self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", "OK") - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4") origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME) # Dump and load without any additional params specified @@ -345,7 +369,7 @@ def testDumpLoad(self): self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath) ) self.assertRunOKFull( - "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4" + "scan --db=%s" % loadedDbPath, "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4" ) # Dump and load in hex @@ -358,7 +382,7 @@ def testDumpLoad(self): ) ) self.assertRunOKFull( - "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4" + "scan --db=%s" % loadedDbPath, "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4" ) # Dump only a portion of the key range @@ -370,7 +394,7 @@ def testDumpLoad(self): self.assertTrue( self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath) ) - self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2") + self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 ==> y1\nx2 ==> y2") # Dump upto max_keys rows dumpFilePath = os.path.join(self.TMP_DIR, "dump4") @@ -379,13 +403,15 @@ def testDumpLoad(self): self.assertTrue( self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath) ) - self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3") + self.assertRunOKFull( + "scan --db=%s" % loadedDbPath, "x1 ==> y1\nx2 ==> y2\nx3 ==> y3" + ) # Load into an existing db, create_if_missing is not specified self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath)) self.assertTrue(self.loadDb("--db=%s" % loadedDbPath, dumpFilePath)) self.assertRunOKFull( - "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4" + "scan --db=%s" % loadedDbPath, "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4" ) # Dump and load with WAL disabled @@ -398,7 +424,7 @@ def testDumpLoad(self): ) ) self.assertRunOKFull( - "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4" + "scan --db=%s" % loadedDbPath, "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4" ) # Dump and load with lots of extra params specified @@ -423,7 +449,7 @@ def testDumpLoad(self): ) ) self.assertRunOKFull( - "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4" + "scan --db=%s" % loadedDbPath, "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4" ) # Dump with count_only @@ -435,7 +461,7 @@ def testDumpLoad(self): ) # DB should have atleast one value for scan to work self.assertRunOKFull("put --db=%s k1 v1" % loadedDbPath, "OK") - self.assertRunOKFull("scan --db=%s" % loadedDbPath, "k1 : v1") + self.assertRunOKFull("scan --db=%s" % loadedDbPath, "k1 ==> v1") # Dump command fails because of typo in params dumpFilePath = os.path.join(self.TMP_DIR, "dump8") @@ -458,7 +484,7 @@ def testDumpLoad(self): ) ) self.assertRunOKFull( - "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4" + "scan --db=%s" % loadedDbPath, "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4" ) blob_files = self.getBlobFiles(loadedDbPath) self.assertTrue(len(blob_files) >= 1) @@ -498,26 +524,26 @@ def testMiscAdminTask(self): # These tests need to be improved; for example with asserts about # whether compaction or level reduction actually took place. self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", "OK") - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4") origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME) self.assertTrue(0 == run_err_null("./ldb compact --db=%s" % origDbPath)) - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4") self.assertTrue( 0 == run_err_null("./ldb reduce_levels --db=%s --new_levels=2" % origDbPath) ) - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4") self.assertTrue( 0 == run_err_null("./ldb reduce_levels --db=%s --new_levels=3" % origDbPath) ) - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4") self.assertTrue( 0 == run_err_null("./ldb compact --db=%s --from=x1 --to=x3" % origDbPath) ) - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4") self.assertTrue( 0 @@ -525,7 +551,7 @@ def testMiscAdminTask(self): "./ldb compact --db=%s --hex --from=0x6131 --to=0x6134" % origDbPath ) ) - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4") # TODO(dilip): Not sure what should be passed to WAL.Currently corrupted. self.assertTrue( @@ -535,7 +561,7 @@ def testMiscAdminTask(self): % (origDbPath, os.path.join(origDbPath, "LOG")) ) ) - self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK("scan", "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4") def testCheckConsistency(self): print("Running testCheckConsistency...") @@ -923,7 +949,9 @@ def testIngestExternalSst(self): "batchput --db=%s --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4" % dbPath, "OK", ) - self.assertRunOK("scan --db=%s" % dbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + self.assertRunOK( + "scan --db=%s" % dbPath, "x1 ==> y1\nx2 ==> y2\nx3 ==> y3\nx4 ==> y4" + ) dumpFilePath = os.path.join(self.TMP_DIR, "dump1") with open(dumpFilePath, "w") as f: f.write("x1 ==> y10\nx2 ==> y20\nx3 ==> y30\nx4 ==> y40") @@ -947,7 +975,7 @@ def testIngestExternalSst(self): ) ) self.assertRunOKFull( - "scan --db=%s" % dbPath, "x1 : y10\nx2 : y20\nx3 : y30\nx4 : y40" + "scan --db=%s" % dbPath, "x1 ==> y10\nx2 ==> y20\nx3 ==> y30\nx4 ==> y40" ) diff --git a/tools/ldb_tool.cc b/tools/ldb_tool.cc index 8d5ac068b..a7aebc121 100644 --- a/tools/ldb_tool.cc +++ b/tools/ldb_tool.cc @@ -10,7 +10,7 @@ namespace ROCKSDB_NAMESPACE { -LDBOptions::LDBOptions() {} +LDBOptions::LDBOptions() = default; void LDBCommandRunner::PrintHelp(const LDBOptions& ldb_options, const char* /*exec_name*/, bool to_stderr) { @@ -180,4 +180,3 @@ void LDBTool::Run(int argc, char** argv, Options options, exit(error_code); } } // namespace ROCKSDB_NAMESPACE - diff --git a/tools/sst_dump_test.cc b/tools/sst_dump_test.cc index 29d11d4da..07c42b6e7 100644 --- a/tools/sst_dump_test.cc +++ b/tools/sst_dump_test.cc @@ -7,8 +7,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include +#include +#include "db/wide/wide_column_serialization.h" #include "file/random_access_file_reader.h" #include "port/stack_trace.h" #include "rocksdb/convenience.h" @@ -24,10 +25,11 @@ namespace ROCKSDB_NAMESPACE { const uint32_t kOptLength = 1024; namespace { -static std::string MakeKey(int i) { +static std::string MakeKey(int i, + ValueType value_type = ValueType::kTypeValue) { char buf[100]; snprintf(buf, sizeof(buf), "k_%04d", i); - InternalKey key(std::string(buf), 0, ValueType::kTypeValue); + InternalKey key(std::string(buf), 0, value_type); return key.Encode().ToString(); } @@ -44,6 +46,16 @@ static std::string MakeValue(int i) { return key.Encode().ToString(); } +static std::string MakeWideColumn(int i) { + std::string val = MakeValue(i); + std::string val1 = "attr_1_val_" + val; + std::string val2 = "attr_2_val_" + val; + WideColumns columns{{"attr_1", val1}, {"attr_2", val2}}; + std::string entity; + EXPECT_OK(WideColumnSerialization::Serialize(columns, entity)); + return entity; +} + void cleanup(const Options& opts, const std::string& file_name) { Env* env = opts.env; ASSERT_OK(env->DeleteFile(file_name)); @@ -94,7 +106,8 @@ class SSTDumpToolTest : public testing::Test { snprintf(usage[2], kOptLength, "--file=%s", file_path.c_str()); } - void createSST(const Options& opts, const std::string& file_name) { + void createSST(const Options& opts, const std::string& file_name, + uint32_t wide_column_one_in = 0) { Env* test_env = opts.env; FileOptions file_options(opts); ReadOptions read_options; @@ -123,7 +136,12 @@ class SSTDumpToolTest : public testing::Test { const char* comparator_name = ikc.user_comparator()->Name(); if (strcmp(comparator_name, ReverseBytewiseComparator()->Name()) == 0) { for (int32_t i = num_keys; i >= 0; i--) { - tb->Add(MakeKey(i), MakeValue(i)); + if (wide_column_one_in == 0 || i % wide_column_one_in != 0) { + tb->Add(MakeKey(i), MakeValue(i)); + } else { + tb->Add(MakeKey(i, ValueType::kTypeWideColumnEntity), + MakeWideColumn(i)); + } } } else if (strcmp(comparator_name, test::BytewiseComparatorWithU64TsWrapper()->Name()) == @@ -133,7 +151,12 @@ class SSTDumpToolTest : public testing::Test { } } else { for (uint32_t i = 0; i < num_keys; i++) { - tb->Add(MakeKey(i), MakeValue(i)); + if (wide_column_one_in == 0 || i % wide_column_one_in != 0) { + tb->Add(MakeKey(i), MakeValue(i)); + } else { + tb->Add(MakeKey(i, ValueType::kTypeWideColumnEntity), + MakeWideColumn(i)); + } } } ASSERT_OK(tb->Finish()); @@ -164,7 +187,7 @@ TEST_F(SSTDumpToolTest, EmptyFilter) { Options opts; opts.env = env(); std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=raw", usage); @@ -212,7 +235,7 @@ TEST_F(SSTDumpToolTest, SstDumpComparatorWithU64Ts) { opts.table_factory.reset(new BlockBasedTableFactory(table_opts)); std::string file_path = MakeFilePath("rocksdb_sst_comparator_with_u64_ts.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=raw", usage); @@ -234,7 +257,7 @@ TEST_F(SSTDumpToolTest, FilterBlock) { ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, true)); opts.table_factory.reset(new BlockBasedTableFactory(table_opts)); std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=raw", usage); @@ -300,7 +323,7 @@ TEST_F(SSTDumpToolTest, CompressedSizes) { ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false)); opts.table_factory.reset(new BlockBasedTableFactory(table_opts)); std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=recompress", usage); @@ -426,7 +449,7 @@ TEST_F(SSTDumpToolTest, RawOutput) { Options opts; opts.env = env(); std::string file_path = MakeFilePath("rocksdb_sst_test.sst"); - createSST(opts, file_path); + createSST(opts, file_path, 10); char* usage[3]; PopulateCommandArgs(file_path, "--command=raw", usage); @@ -468,4 +491,3 @@ int main(int argc, char** argv) { RegisterCustomObjects(argc, argv); return RUN_ALL_TESTS(); } - diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc index 1b269043a..59df1af8d 100644 --- a/tools/sst_dump_tool.cc +++ b/tools/sst_dump_tool.cc @@ -400,7 +400,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { // that whether it is a valid sst or not // (A directory "file" is not a valid sst) filenames.clear(); - filenames.push_back(dir_or_file); + filenames.emplace_back(dir_or_file); dir = false; } @@ -468,7 +468,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { fprintf(stderr, "%s: %s\n", filename.c_str(), st.ToString().c_str()); exit(1); } else { - fprintf(stdout, "raw dump written to file %s\n", &out_filename[0]); + fprintf(stdout, "raw dump written to file %s\n", out_filename.data()); } continue; } diff --git a/tools/trace_analyzer_test.cc b/tools/trace_analyzer_test.cc index 81dc4f2cc..1d5c87054 100644 --- a/tools/trace_analyzer_test.cc +++ b/tools/trace_analyzer_test.cc @@ -31,6 +31,8 @@ int main() { #include "test_util/testutil.h" #include "tools/trace_analyzer_tool.h" #include "trace_replay/trace_replay.h" +#include "utilities/fault_injection_env.h" +#include "utilities/trace/file_trace_reader_writer.h" namespace ROCKSDB_NAMESPACE { @@ -54,7 +56,7 @@ class TraceAnalyzerTest : public testing::Test { dbname_ = test_path_ + "/db"; } - ~TraceAnalyzerTest() override {} + ~TraceAnalyzerTest() override = default; void GenerateTrace(std::string trace_path) { Options options; @@ -85,11 +87,11 @@ class TraceAnalyzerTest : public testing::Test { ASSERT_OK(batch.DeleteRange("e", "f")); ASSERT_OK(db_->Write(wo, &batch)); std::vector keys; - keys.push_back("a"); - keys.push_back("b"); - keys.push_back("df"); - keys.push_back("gege"); - keys.push_back("hjhjhj"); + keys.emplace_back("a"); + keys.emplace_back("b"); + keys.emplace_back("df"); + keys.emplace_back("gege"); + keys.emplace_back("hjhjhj"); std::vector values; std::vector ss = db_->MultiGet(ro, keys, &values); ASSERT_GE(ss.size(), 0); @@ -174,8 +176,6 @@ class TraceAnalyzerTest : public testing::Test { ASSERT_EQ(result[i][0], cnt[i][0]); } } - - return; } void AnalyzeTrace(std::vector& paras_diff, @@ -785,6 +785,48 @@ TEST_F(TraceAnalyzerTest, Iterator) { */ } +TEST_F(TraceAnalyzerTest, ExistsPreviousTraceWriteError) { + DB* db_ = nullptr; + Options options; + options.create_if_missing = true; + + std::unique_ptr fault_env( + new FaultInjectionTestEnv(env_)); + const std::string trace_path = + test_path_ + "/previous_trace_write_error_trace"; + std::unique_ptr trace_writer; + ASSERT_OK(NewFileTraceWriter(fault_env.get(), env_options_, trace_path, + &trace_writer)); + + ASSERT_OK(DB::Open(options, dbname_, &db_)); + ASSERT_OK(db_->StartTrace(TraceOptions(), std::move(trace_writer))); + + // Inject write error on the first trace write. + // This trace write is made big enough to actually write to FS for error + // injection. + const std::string kBigKey(1000000, 'k'); + const std::string kBigValue(1000000, 'v'); + fault_env->SetFilesystemActive(false, Status::IOError("Injected")); + + ASSERT_OK(db_->Put(WriteOptions(), kBigKey, kBigValue)); + + fault_env->SetFilesystemActive(true); + + // Without proper handling of the previous trace write error, + // this trace write will continue and crash the db (in DEBUG_LEVEL > 0) + // due to writing to the trace file that has seen error. + ASSERT_OK(db_->Put(WriteOptions(), kBigKey, kBigValue)); + + // Verify `EndTrace()` returns the previous write trace error if any + Status s = db_->EndTrace(); + ASSERT_TRUE(s.IsIncomplete()); + ASSERT_TRUE(s.ToString().find("Tracing has seen error") != std::string::npos); + ASSERT_TRUE(s.ToString().find("Injected") != std::string::npos); + + delete db_; + ASSERT_OK(DestroyDB(dbname_, options)); +} + // Test analyzing of multiget TEST_F(TraceAnalyzerTest, MultiGet) { std::string trace_path = test_path_ + "/trace"; diff --git a/tools/trace_analyzer_tool.cc b/tools/trace_analyzer_tool.cc index 00a4da046..a1e321dcd 100644 --- a/tools/trace_analyzer_tool.cc +++ b/tools/trace_analyzer_tool.cc @@ -201,7 +201,7 @@ uint64_t MultiplyCheckOverflow(uint64_t op1, uint64_t op2) { AnalyzerOptions::AnalyzerOptions() : correlation_map(kTaTypeNum, std::vector(kTaTypeNum, -1)) {} -AnalyzerOptions::~AnalyzerOptions() {} +AnalyzerOptions::~AnalyzerOptions() = default; void AnalyzerOptions::SparseCorrelationInput(const std::string& in_str) { std::string cur = in_str; @@ -214,14 +214,14 @@ void AnalyzerOptions::SparseCorrelationInput(const std::string& in_str) { exit(1); } std::string opt1, opt2; - std::size_t split = cur.find_first_of(","); + std::size_t split = cur.find_first_of(','); if (split != std::string::npos) { opt1 = cur.substr(1, split - 1); } else { fprintf(stderr, "Invalid correlation input: %s\n", in_str.c_str()); exit(1); } - std::size_t end = cur.find_first_of("]"); + std::size_t end = cur.find_first_of(']'); if (end != std::string::npos) { opt2 = cur.substr(split + 1, end - split - 1); } else { @@ -232,8 +232,7 @@ void AnalyzerOptions::SparseCorrelationInput(const std::string& in_str) { if (taOptToIndex.find(opt1) != taOptToIndex.end() && taOptToIndex.find(opt2) != taOptToIndex.end()) { - correlation_list.push_back( - std::make_pair(taOptToIndex[opt1], taOptToIndex[opt2])); + correlation_list.emplace_back(taOptToIndex[opt1], taOptToIndex[opt2]); } else { fprintf(stderr, "Invalid correlation input: %s\n", in_str.c_str()); exit(1); @@ -245,7 +244,6 @@ void AnalyzerOptions::SparseCorrelationInput(const std::string& in_str) { correlation_map[it.first][it.second] = sequence; sequence++; } - return; } // The trace statistic struct constructor @@ -264,7 +262,7 @@ TraceStats::TraceStats() { a_ave_qps = 0.0; } -TraceStats::~TraceStats() {} +TraceStats::~TraceStats() = default; // The trace analyzer constructor TraceAnalyzer::TraceAnalyzer(std::string& trace_path, std::string& output_path, @@ -354,7 +352,7 @@ TraceAnalyzer::TraceAnalyzer(std::string& trace_path, std::string& output_path, } } -TraceAnalyzer::~TraceAnalyzer() {} +TraceAnalyzer::~TraceAnalyzer() = default; // Prepare the processing // Initiate the global trace reader and writer here diff --git a/trace_replay/trace_record.cc b/trace_replay/trace_record.cc index 21df0275d..a4a4eb9f8 100644 --- a/trace_replay/trace_record.cc +++ b/trace_replay/trace_record.cc @@ -97,7 +97,7 @@ IteratorQueryTraceRecord::IteratorQueryTraceRecord( upper_.PinSelf(upper_bound); } -IteratorQueryTraceRecord::~IteratorQueryTraceRecord() {} +IteratorQueryTraceRecord::~IteratorQueryTraceRecord() = default; Slice IteratorQueryTraceRecord::GetLowerBound() const { return Slice(lower_); } diff --git a/trace_replay/trace_replay.cc b/trace_replay/trace_replay.cc index c681e374c..6ade8e316 100644 --- a/trace_replay/trace_replay.cc +++ b/trace_replay/trace_replay.cc @@ -58,8 +58,8 @@ Status TracerHelper::ParseTraceHeader(const Trace& header, int* trace_version, std::vector s_vec; int begin = 0, end; for (int i = 0; i < 3; i++) { - assert(header.payload.find("\t", begin) != std::string::npos); - end = static_cast(header.payload.find("\t", begin)); + assert(header.payload.find('\t', begin) != std::string::npos); + end = static_cast(header.payload.find('\t', begin)); s_vec.push_back(header.payload.substr(begin, end - begin)); begin = end + 1; } @@ -345,7 +345,8 @@ Tracer::Tracer(SystemClock* clock, const TraceOptions& trace_options, : clock_(clock), trace_options_(trace_options), trace_writer_(std::move(trace_writer)), - trace_request_count_(0) { + trace_request_count_(0), + trace_write_status_(Status::OK()) { // TODO: What if this fails? WriteHeader().PermitUncheckedError(); } @@ -612,9 +613,18 @@ Status Tracer::WriteFooter() { } Status Tracer::WriteTrace(const Trace& trace) { + if (!trace_write_status_.ok()) { + return Status::Incomplete("Tracing has seen error: %s", + trace_write_status_.ToString()); + } + assert(trace_write_status_.ok()); std::string encoded_trace; TracerHelper::EncodeTrace(trace, &encoded_trace); - return trace_writer_->Write(Slice(encoded_trace)); + Status s = trace_writer_->Write(Slice(encoded_trace)); + if (!s.ok()) { + trace_write_status_ = s; + } + return s; } Status Tracer::Close() { return WriteFooter(); } diff --git a/trace_replay/trace_replay.h b/trace_replay/trace_replay.h index 9aba5ceb7..55908dcb7 100644 --- a/trace_replay/trace_replay.h +++ b/trace_replay/trace_replay.h @@ -178,6 +178,7 @@ class Tracer { TraceOptions trace_options_; std::unique_ptr trace_writer_; uint64_t trace_request_count_; + Status trace_write_status_; }; } // namespace ROCKSDB_NAMESPACE diff --git a/unreleased_history/behavior_changes/debt_based_speedup.md b/unreleased_history/behavior_changes/debt_based_speedup.md new file mode 100644 index 000000000..8db8b6688 --- /dev/null +++ b/unreleased_history/behavior_changes/debt_based_speedup.md @@ -0,0 +1 @@ +Compactions can be scheduled in parallel in an additional scenario: high compaction debt relative to the data size diff --git a/unreleased_history/behavior_changes/marking_based_speedup.md b/unreleased_history/behavior_changes/marking_based_speedup.md new file mode 100644 index 000000000..cf01b8f58 --- /dev/null +++ b/unreleased_history/behavior_changes/marking_based_speedup.md @@ -0,0 +1 @@ +Compactions can be scheduled in parallel in an additional scenario: multiple files are marked for compaction within a single column family diff --git a/unreleased_history/behavior_changes/more_debt_based_speedup.md b/unreleased_history/behavior_changes/more_debt_based_speedup.md new file mode 100644 index 000000000..c10f0d7b3 --- /dev/null +++ b/unreleased_history/behavior_changes/more_debt_based_speedup.md @@ -0,0 +1 @@ +Reduced the compaction debt ratio trigger for scheduling parallel compactions diff --git a/unreleased_history/bug_fixes/fix-import-epoch.md b/unreleased_history/bug_fixes/fix-import-epoch.md new file mode 100644 index 000000000..97df6276f --- /dev/null +++ b/unreleased_history/bug_fixes/fix-import-epoch.md @@ -0,0 +1 @@ +* Fix a bug in CreateColumnFamilyWithImport() where if multiple CFs are imported, we were not resetting files' epoch number and L0 files can have overlapping key range but the same epoch number. \ No newline at end of file diff --git a/unreleased_history/bug_fixes/new_ingested_data_with_old_seqno.md b/unreleased_history/bug_fixes/new_ingested_data_with_old_seqno.md new file mode 100644 index 000000000..8d0f32b4b --- /dev/null +++ b/unreleased_history/bug_fixes/new_ingested_data_with_old_seqno.md @@ -0,0 +1 @@ +Fix a bug where older data of an ingested key can be returned for read when universal compaction is used diff --git a/unreleased_history/release.sh b/unreleased_history/release.sh index 5ddbe3bfe..1f50f51b0 100755 --- a/unreleased_history/release.sh +++ b/unreleased_history/release.sh @@ -34,8 +34,12 @@ awk '/#define ROCKSDB_MAJOR/ { major = $3 } echo " (`git log -n1 --date=format:"%m/%d/%Y" --format="%ad"`)" >> HISTORY.new function process_file () { - # use awk to correct extra or missing newlines, missing '* ' on first line - awk '/./ { if (notfirstline || $1 == "*") print; + # use awk to correct + # * extra or missing newlines + # * leading or trailing whitespace + # * missing '* ' on first line + awk '/./ { gsub(/^[ \t]+/, ""); gsub(/[ \t]+$/, ""); + if (notfirstline || $1 == "*") print; else print "* " $0; notfirstline=1; }' < $1 >> HISTORY.new echo git rm $1 diff --git a/util/atomic.h b/util/atomic.h new file mode 100644 index 000000000..afb3dc540 --- /dev/null +++ b/util/atomic.h @@ -0,0 +1,111 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// Background: +// std::atomic is somewhat easy to misuse: +// * Implicit conversion to T using std::memory_order_seq_cst, along with +// memory order parameter defaults, make it easy to accidentally mix sequential +// consistency ordering with acquire/release memory ordering. See +// "The single total order might not be consistent with happens-before" at +// https://en.cppreference.com/w/cpp/atomic/memory_order +// * It's easy to use nonsensical (UB) combinations like store with +// std::memory_order_acquire. +// For such reasons, we provide wrappers below to make safe usage easier. + +// Wrapper around std::atomic to avoid certain bugs (see Background above). +// +// This relaxed-only wrapper is intended for atomics that do not need +// ordering constraints with other data reads/writes aside from those +// necessary for computing data values or given by other happens-before +// relationships. For example, a cross-thread counter that never returns +// the same result can be a RelaxedAtomic. +template +class RelaxedAtomic { + public: + explicit RelaxedAtomic(T initial = {}) : v_(initial) {} + void StoreRelaxed(T desired) { v_.store(desired, std::memory_order_relaxed); } + T LoadRelaxed() const { return v_.load(std::memory_order_relaxed); } + bool CasWeakRelaxed(T& expected, T desired) { + return v_.compare_exchange_weak(expected, desired, + std::memory_order_relaxed); + } + bool CasStrongRelaxed(T& expected, T desired) { + return v_.compare_exchange_strong(expected, desired, + std::memory_order_relaxed); + } + T ExchangeRelaxed(T desired) { + return v_.exchange(desired, std::memory_order_relaxed); + } + T FetchAddRelaxed(T operand) { + return v_.fetch_add(operand, std::memory_order_relaxed); + } + T FetchSubRelaxed(T operand) { + return v_.fetch_sub(operand, std::memory_order_relaxed); + } + T FetchAndRelaxed(T operand) { + return v_.fetch_and(operand, std::memory_order_relaxed); + } + T FetchOrRelaxed(T operand) { + return v_.fetch_or(operand, std::memory_order_relaxed); + } + T FetchXorRelaxed(T operand) { + return v_.fetch_xor(operand, std::memory_order_relaxed); + } + + protected: + std::atomic v_; +}; + +// Wrapper around std::atomic to avoid certain bugs (see Background above). +// +// Except for some unusual cases requiring sequential consistency, this is +// a general-purpose atomic. Relaxed operations can be mixed in as appropriate. +template +class AcqRelAtomic : public RelaxedAtomic { + public: + explicit AcqRelAtomic(T initial = {}) : RelaxedAtomic(initial) {} + void Store(T desired) { + RelaxedAtomic::v_.store(desired, std::memory_order_release); + } + T Load() const { + return RelaxedAtomic::v_.load(std::memory_order_acquire); + } + bool CasWeak(T& expected, T desired) { + return RelaxedAtomic::v_.compare_exchange_weak( + expected, desired, std::memory_order_acq_rel); + } + bool CasStrong(T& expected, T desired) { + return RelaxedAtomic::v_.compare_exchange_strong( + expected, desired, std::memory_order_acq_rel); + } + T Exchange(T desired) { + return RelaxedAtomic::v_.exchange(desired, std::memory_order_acq_rel); + } + T FetchAdd(T operand) { + return RelaxedAtomic::v_.fetch_add(operand, std::memory_order_acq_rel); + } + T FetchSub(T operand) { + return RelaxedAtomic::v_.fetch_sub(operand, std::memory_order_acq_rel); + } + T FetchAnd(T operand) { + return RelaxedAtomic::v_.fetch_and(operand, std::memory_order_acq_rel); + } + T FetchOr(T operand) { + return RelaxedAtomic::v_.fetch_or(operand, std::memory_order_acq_rel); + } + T FetchXor(T operand) { + return RelaxedAtomic::v_.fetch_xor(operand, std::memory_order_acq_rel); + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/util/autovector.h b/util/autovector.h index 79ee5de57..39c7aabee 100644 --- a/util/autovector.h +++ b/util/autovector.h @@ -61,7 +61,7 @@ class autovector { using iterator_category = std::random_access_iterator_tag; iterator_impl(TAutoVector* vect, size_t index) - : vect_(vect), index_(index){}; + : vect_(vect), index_(index){} iterator_impl(const iterator_impl&) = default; ~iterator_impl() {} iterator_impl& operator=(const iterator_impl&) = default; diff --git a/util/bloom_impl.h b/util/bloom_impl.h index 53b70aa68..c9bbb125b 100644 --- a/util/bloom_impl.h +++ b/util/bloom_impl.h @@ -199,7 +199,7 @@ class FastLocalBloomImpl { static inline void AddHash(uint32_t h1, uint32_t h2, uint32_t len_bytes, int num_probes, char *data) { - uint32_t bytes_to_cache_line = FastRange32(len_bytes >> 6, h1) << 6; + uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6; AddHashPrepared(h2, num_probes, data + bytes_to_cache_line); } @@ -216,7 +216,7 @@ class FastLocalBloomImpl { static inline void PrepareHash(uint32_t h1, uint32_t len_bytes, const char *data, uint32_t /*out*/ *byte_offset) { - uint32_t bytes_to_cache_line = FastRange32(len_bytes >> 6, h1) << 6; + uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6; PREFETCH(data + bytes_to_cache_line, 0 /* rw */, 1 /* locality */); PREFETCH(data + bytes_to_cache_line + 63, 0 /* rw */, 1 /* locality */); *byte_offset = bytes_to_cache_line; @@ -224,7 +224,7 @@ class FastLocalBloomImpl { static inline bool HashMayMatch(uint32_t h1, uint32_t h2, uint32_t len_bytes, int num_probes, const char *data) { - uint32_t bytes_to_cache_line = FastRange32(len_bytes >> 6, h1) << 6; + uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6; return HashMayMatchPrepared(h2, num_probes, data + bytes_to_cache_line); } diff --git a/util/bloom_test.cc b/util/bloom_test.cc index 06dd1de06..b0a5cae56 100644 --- a/util/bloom_test.cc +++ b/util/bloom_test.cc @@ -23,6 +23,7 @@ int main() { #include "cache/cache_reservation_manager.h" #include "memory/arena.h" #include "port/jemalloc_helper.h" +#include "rocksdb/convenience.h" #include "rocksdb/filter_policy.h" #include "table/block_based/filter_policy_internal.h" #include "test_util/testharness.h" @@ -1109,12 +1110,16 @@ static void SetTestingLevel(int levelish, FilterBuildingContext* ctx) { TEST(RibbonTest, RibbonTestLevelThreshold) { BlockBasedTableOptions opts; FilterBuildingContext ctx(opts); + + std::shared_ptr reused{NewRibbonFilterPolicy(10)}; + // A few settings for (CompactionStyle cs : {kCompactionStyleLevel, kCompactionStyleUniversal, kCompactionStyleFIFO, kCompactionStyleNone}) { ctx.compaction_style = cs; - for (int bloom_before_level : {-1, 0, 1, 10}) { - std::vector > policies; + for (int bloom_before_level : {-1, 0, 1, 10, INT_MAX - 1, INT_MAX}) { + SCOPED_TRACE("bloom_before_level=" + std::to_string(bloom_before_level)); + std::vector > policies; policies.emplace_back(NewRibbonFilterPolicy(10, bloom_before_level)); if (bloom_before_level == 0) { @@ -1122,16 +1127,22 @@ TEST(RibbonTest, RibbonTestLevelThreshold) { policies.emplace_back(NewRibbonFilterPolicy(10)); } - for (std::unique_ptr& policy : policies) { - // Claim to be generating filter for this level - SetTestingLevel(bloom_before_level, &ctx); + ASSERT_OK(reused->ConfigureOption({}, "bloom_before_level", + std::to_string(bloom_before_level))); - std::unique_ptr builder{ - policy->GetBuilderWithContext(ctx)}; + policies.push_back(reused); - // Must be Ribbon (more space efficient than 10 bits per key) - ASSERT_LT(GetEffectiveBitsPerKey(builder.get()), 8); + for (auto& policy : policies) { + std::unique_ptr builder; + if (bloom_before_level < INT_MAX) { + // Claim to be generating filter for this level + SetTestingLevel(bloom_before_level, &ctx); + + builder.reset(policy->GetBuilderWithContext(ctx)); + // Must be Ribbon (more space efficient than 10 bits per key) + ASSERT_LT(GetEffectiveBitsPerKey(builder.get()), 8); + } if (bloom_before_level >= 0) { // Claim to be generating filter for previous level SetTestingLevel(bloom_before_level - 1, &ctx); @@ -1142,6 +1153,10 @@ TEST(RibbonTest, RibbonTestLevelThreshold) { // Level is considered. // Must be Bloom (~ 10 bits per key) ASSERT_GT(GetEffectiveBitsPerKey(builder.get()), 9); + } else if (bloom_before_level == INT_MAX) { + // Force bloom option + // Must be Bloom (~ 10 bits per key) + ASSERT_GT(GetEffectiveBitsPerKey(builder.get()), 9); } else { // Level is ignored under non-traditional compaction styles. // Must be Ribbon (more space efficient than 10 bits per key) @@ -1155,8 +1170,14 @@ TEST(RibbonTest, RibbonTestLevelThreshold) { builder.reset(policy->GetBuilderWithContext(ctx)); - // Must be Ribbon (more space efficient than 10 bits per key) - ASSERT_LT(GetEffectiveBitsPerKey(builder.get()), 8); + if (bloom_before_level < INT_MAX) { + // Must be Ribbon (more space efficient than 10 bits per key) + ASSERT_LT(GetEffectiveBitsPerKey(builder.get()), 8); + } else { + // Force bloom option + // Must be Bloom (~ 10 bits per key) + ASSERT_GT(GetEffectiveBitsPerKey(builder.get()), 9); + } } } } diff --git a/util/cast_util.h b/util/cast_util.h index 3c381d9b2..e010274a7 100644 --- a/util/cast_util.h +++ b/util/cast_util.h @@ -5,6 +5,7 @@ #pragma once +#include #include #include @@ -53,4 +54,13 @@ inline To lossless_cast(From x) { return static_cast(x); } +// For disambiguating a potentially heterogeneous aggregate as a homogeneous +// initializer list. E.g. might be able to write List({x, y}) in some cases +// instead of std::vector({x, y}). +template +inline const std::initializer_list& List( + const std::initializer_list& list) { + return list; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/util/compaction_job_stats_impl.cc b/util/compaction_job_stats_impl.cc index 587a26f24..cdb591f23 100644 --- a/util/compaction_job_stats_impl.cc +++ b/util/compaction_job_stats_impl.cc @@ -12,6 +12,7 @@ void CompactionJobStats::Reset() { elapsed_micros = 0; cpu_micros = 0; + has_num_input_records = true; num_input_records = 0; num_blobs_read = 0; num_input_files = 0; @@ -55,6 +56,7 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) { elapsed_micros += stats.elapsed_micros; cpu_micros += stats.cpu_micros; + has_num_input_records &= stats.has_num_input_records; num_input_records += stats.num_input_records; num_blobs_read += stats.num_blobs_read; num_input_files += stats.num_input_files; diff --git a/util/comparator.cc b/util/comparator.cc index 19fd47387..a5d7a7ca0 100644 --- a/util/comparator.cc +++ b/util/comparator.cc @@ -9,9 +9,8 @@ #include "rocksdb/comparator.h" -#include - #include +#include #include #include #include @@ -23,13 +22,14 @@ #include "rocksdb/slice.h" #include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/object_registry.h" +#include "util/coding.h" namespace ROCKSDB_NAMESPACE { namespace { class BytewiseComparatorImpl : public Comparator { public: - BytewiseComparatorImpl() {} + BytewiseComparatorImpl() = default; static const char* kClassName() { return "leveldb.BytewiseComparator"; } const char* Name() const override { return kClassName(); } @@ -111,7 +111,9 @@ class BytewiseComparatorImpl : public Comparator { } size_t diff_ind = s.difference_offset(t); // same slice - if (diff_ind >= s.size()) return false; + if (diff_ind >= s.size()) { + return false; + } uint8_t byte_s = static_cast(s[diff_ind]); uint8_t byte_t = static_cast(t[diff_ind]); // first different byte must be consecutive, and remaining bytes must be @@ -147,7 +149,7 @@ class BytewiseComparatorImpl : public Comparator { class ReverseBytewiseComparatorImpl : public BytewiseComparatorImpl { public: - ReverseBytewiseComparatorImpl() {} + ReverseBytewiseComparatorImpl() = default; static const char* kClassName() { return "rocksdb.ReverseBytewiseComparator"; @@ -230,7 +232,6 @@ class ReverseBytewiseComparatorImpl : public BytewiseComparatorImpl { } }; -// EXPERIMENTAL // Comparator with 64-bit integer timestamp. // We did not performance test this yet. template @@ -250,6 +251,12 @@ class ComparatorWithU64TsImpl : public Comparator { const char* Name() const override { return kClassName(); } + // The comparator that compares the user key without timestamp part is treated + // as the root comparator. + const Comparator* GetRootComparator() const override { + return &cmp_without_ts_; + } + void FindShortSuccessor(std::string*) const override {} void FindShortestSeparator(std::string*, const Slice&) const override {} int Compare(const Slice& a, const Slice& b) const override { @@ -316,24 +323,62 @@ const Comparator* BytewiseComparatorWithU64Ts() { return &comp_with_u64_ts; } +const Comparator* ReverseBytewiseComparatorWithU64Ts() { + STATIC_AVOID_DESTRUCTION( + ComparatorWithU64TsImpl, comp_with_u64_ts); + return &comp_with_u64_ts; +} + +Status DecodeU64Ts(const Slice& ts, uint64_t* int_ts) { + if (ts.size() != sizeof(uint64_t)) { + return Status::InvalidArgument("U64Ts timestamp size mismatch."); + } + *int_ts = DecodeFixed64(ts.data()); + return Status::OK(); +} + +Slice EncodeU64Ts(uint64_t ts, std::string* ts_buf) { + char buf[sizeof(ts)]; + EncodeFixed64(buf, ts); + ts_buf->assign(buf, sizeof(buf)); + return Slice(*ts_buf); +} + +Slice MaxU64Ts() { + static constexpr char kTsMax[] = "\xff\xff\xff\xff\xff\xff\xff\xff"; + return Slice(kTsMax, sizeof(uint64_t)); +} + +Slice MinU64Ts() { + static constexpr char kTsMin[] = "\x00\x00\x00\x00\x00\x00\x00\x00"; + return Slice(kTsMin, sizeof(uint64_t)); +} + static int RegisterBuiltinComparators(ObjectLibrary& library, const std::string& /*arg*/) { library.AddFactory( BytewiseComparatorImpl::kClassName(), [](const std::string& /*uri*/, - std::unique_ptr* /*guard */, - std::string* /* errmsg */) { return BytewiseComparator(); }); + std::unique_ptr* /*guard*/, + std::string* /*errmsg*/) { return BytewiseComparator(); }); library.AddFactory( ReverseBytewiseComparatorImpl::kClassName(), [](const std::string& /*uri*/, - std::unique_ptr* /*guard */, - std::string* /* errmsg */) { return ReverseBytewiseComparator(); }); + std::unique_ptr* /*guard*/, + std::string* /*errmsg*/) { return ReverseBytewiseComparator(); }); library.AddFactory( ComparatorWithU64TsImpl::kClassName(), [](const std::string& /*uri*/, - std::unique_ptr* /*guard */, - std::string* /* errmsg */) { return BytewiseComparatorWithU64Ts(); }); - return 3; + std::unique_ptr* /*guard*/, + std::string* /*errmsg*/) { return BytewiseComparatorWithU64Ts(); }); + library.AddFactory( + ComparatorWithU64TsImpl::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* /*guard*/, + std::string* /*errmsg*/) { + return ReverseBytewiseComparatorWithU64Ts(); + }); + return 4; } Status Comparator::CreateFromString(const ConfigOptions& config_options, @@ -357,6 +402,9 @@ Status Comparator::CreateFromString(const ConfigOptions& config_options, } else if (id == ComparatorWithU64TsImpl::kClassName()) { *result = BytewiseComparatorWithU64Ts(); + } else if (id == ComparatorWithU64TsImpl< + ReverseBytewiseComparatorImpl>::kClassName()) { + *result = ReverseBytewiseComparatorWithU64Ts(); } else if (value.empty()) { // No Id and no options. Clear the object *result = nullptr; diff --git a/util/compression.cc b/util/compression.cc index 712d333ee..2a0bc38d4 100644 --- a/util/compression.cc +++ b/util/compression.cc @@ -48,7 +48,7 @@ int ZSTDStreamingCompress::Compress(const char* input, size_t input_size, if (input_size == 0) { return 0; } -#ifndef ZSTD_STREAMING +#ifndef ZSTD_ADVANCED (void)input; (void)input_size; (void)output; @@ -77,7 +77,7 @@ int ZSTDStreamingCompress::Compress(const char* input, size_t input_size, } void ZSTDStreamingCompress::Reset() { -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED ZSTD_CCtx_reset(cctx_, ZSTD_ResetDirective::ZSTD_reset_session_only); input_buffer_ = {/*src=*/nullptr, /*size=*/0, /*pos=*/0}; #endif @@ -91,7 +91,7 @@ int ZSTDStreamingUncompress::Uncompress(const char* input, size_t input_size, if (input_size == 0) { return 0; } -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED if (input) { // New input input_buffer_ = {input, input_size, /*pos=*/0}; @@ -113,7 +113,7 @@ int ZSTDStreamingUncompress::Uncompress(const char* input, size_t input_size, } void ZSTDStreamingUncompress::Reset() { -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED ZSTD_DCtx_reset(dctx_, ZSTD_ResetDirective::ZSTD_reset_session_only); input_buffer_ = {/*src=*/nullptr, /*size=*/0, /*pos=*/0}; #endif diff --git a/util/compression.h b/util/compression.h index eb5d1c7bb..5620969d7 100644 --- a/util/compression.h +++ b/util/compression.h @@ -53,8 +53,11 @@ #include #endif // ZSTD_VERSION_NUMBER >= 10103 // v1.4.0+ +// ZSTD_Compress2(), ZSTD_compressStream2() and frame parameters all belong to +// advanced APIs and require v1.4.0+. +// https://github.com/facebook/zstd/blob/eb9f881eb810f2242f1ef36b3f3e7014eecb8fa6/lib/zstd.h#L297C40-L297C45 #if ZSTD_VERSION_NUMBER >= 10400 -#define ZSTD_STREAMING +#define ZSTD_ADVANCED #endif // ZSTD_VERSION_NUMBER >= 10400 namespace ROCKSDB_NAMESPACE { // Need this for the context allocation override @@ -180,6 +183,9 @@ struct CompressionDict { if (level == CompressionOptions::kDefaultCompressionLevel) { // 3 is the value of ZSTD_CLEVEL_DEFAULT (not exposed publicly), see // https://github.com/facebook/zstd/issues/1148 + // TODO(cbi): ZSTD_CLEVEL_DEFAULT is exposed after + // https://github.com/facebook/zstd/pull/1174. Use ZSTD_CLEVEL_DEFAULT + // instead of hardcoding 3. level = 3; } // Should be safe (but slower) if below call fails as we'll use the @@ -363,14 +369,43 @@ class CompressionContext { private: #if defined(ZSTD) && (ZSTD_VERSION_NUMBER >= 500) ZSTD_CCtx* zstd_ctx_ = nullptr; - void CreateNativeContext(CompressionType type) { - if (type == kZSTD || type == kZSTDNotFinalCompression) { + + ZSTD_CCtx* CreateZSTDContext() { #ifdef ROCKSDB_ZSTD_CUSTOM_MEM - zstd_ctx_ = - ZSTD_createCCtx_advanced(port::GetJeZstdAllocationOverrides()); + return ZSTD_createCCtx_advanced(port::GetJeZstdAllocationOverrides()); #else // ROCKSDB_ZSTD_CUSTOM_MEM - zstd_ctx_ = ZSTD_createCCtx(); + return ZSTD_createCCtx(); #endif // ROCKSDB_ZSTD_CUSTOM_MEM + } + + void CreateNativeContext(CompressionType type, int level, bool checksum) { + if (type == kZSTD || type == kZSTDNotFinalCompression) { + zstd_ctx_ = CreateZSTDContext(); +#ifdef ZSTD_ADVANCED + if (level == CompressionOptions::kDefaultCompressionLevel) { + // 3 is the value of ZSTD_CLEVEL_DEFAULT (not exposed publicly), see + // https://github.com/facebook/zstd/issues/1148 + level = 3; + } + size_t err = + ZSTD_CCtx_setParameter(zstd_ctx_, ZSTD_c_compressionLevel, level); + if (ZSTD_isError(err)) { + assert(false); + ZSTD_freeCCtx(zstd_ctx_); + zstd_ctx_ = CreateZSTDContext(); + } + if (checksum) { + err = ZSTD_CCtx_setParameter(zstd_ctx_, ZSTD_c_checksumFlag, 1); + if (ZSTD_isError(err)) { + assert(false); + ZSTD_freeCCtx(zstd_ctx_); + zstd_ctx_ = CreateZSTDContext(); + } + } +#else + (void)level; + (void)checksum; +#endif } } void DestroyNativeContext() { @@ -388,12 +423,14 @@ class CompressionContext { #else // ZSTD && (ZSTD_VERSION_NUMBER >= 500) private: - void CreateNativeContext(CompressionType /* type */) {} + void CreateNativeContext(CompressionType /* type */, int /* level */, + bool /* checksum */) {} void DestroyNativeContext() {} #endif // ZSTD && (ZSTD_VERSION_NUMBER >= 500) public: - explicit CompressionContext(CompressionType type) { - CreateNativeContext(type); + explicit CompressionContext(CompressionType type, + const CompressionOptions& options) { + CreateNativeContext(type, options.level, options.checksum); } ~CompressionContext() { DestroyNativeContext(); } CompressionContext(const CompressionContext&) = delete; @@ -525,7 +562,7 @@ inline bool ZSTDNotFinal_Supported() { } inline bool ZSTD_Streaming_Supported() { -#if defined(ZSTD) && defined(ZSTD_STREAMING) +#if defined(ZSTD_ADVANCED) return true; #else return false; @@ -1116,9 +1153,15 @@ inline bool LZ4_Compress(const CompressionInfo& info, static_cast(compression_dict.size())); } #if LZ4_VERSION_NUMBER >= 10700 // r129+ - outlen = - LZ4_compress_fast_continue(stream, input, &(*output)[output_header_len], - static_cast(length), compress_bound, 1); + int acceleration; + if (info.options().level < 0) { + acceleration = -info.options().level; + } else { + acceleration = 1; + } + outlen = LZ4_compress_fast_continue( + stream, input, &(*output)[output_header_len], static_cast(length), + compress_bound, acceleration); #else // up to r128 outlen = LZ4_compress_limitedOutput_continue( stream, input, &(*output)[output_header_len], static_cast(length), @@ -1343,30 +1386,44 @@ inline bool ZSTD_Compress(const CompressionInfo& info, const char* input, size_t compressBound = ZSTD_compressBound(length); output->resize(static_cast(output_header_len + compressBound)); size_t outlen = 0; - int level; - if (info.options().level == CompressionOptions::kDefaultCompressionLevel) { - // 3 is the value of ZSTD_CLEVEL_DEFAULT (not exposed publicly), see - // https://github.com/facebook/zstd/issues/1148 - level = 3; - } else { - level = info.options().level; - } #if ZSTD_VERSION_NUMBER >= 500 // v0.5.0+ ZSTD_CCtx* context = info.context().ZSTDPreallocCtx(); assert(context != nullptr); +#ifdef ZSTD_ADVANCED + if (info.dict().GetDigestedZstdCDict() != nullptr) { + ZSTD_CCtx_refCDict(context, info.dict().GetDigestedZstdCDict()); + } else { + ZSTD_CCtx_loadDictionary(context, info.dict().GetRawDict().data(), + info.dict().GetRawDict().size()); + } + + // Compression level is set in `contex` during CreateNativeContext() + outlen = ZSTD_compress2(context, &(*output)[output_header_len], compressBound, + input, length); +#else // ZSTD_ADVANCED #if ZSTD_VERSION_NUMBER >= 700 // v0.7.0+ if (info.dict().GetDigestedZstdCDict() != nullptr) { outlen = ZSTD_compress_usingCDict(context, &(*output)[output_header_len], compressBound, input, length, info.dict().GetDigestedZstdCDict()); } -#endif // ZSTD_VERSION_NUMBER >= 700 +#endif // ZSTD_VERSION_NUMBER >= 700 + // TODO (cbi): error handling for compression. if (outlen == 0) { + int level; + if (info.options().level == CompressionOptions::kDefaultCompressionLevel) { + // 3 is the value of ZSTD_CLEVEL_DEFAULT (not exposed publicly), see + // https://github.com/facebook/zstd/issues/1148 + level = 3; + } else { + level = info.options().level; + } outlen = ZSTD_compress_usingDict(context, &(*output)[output_header_len], compressBound, input, length, info.dict().GetRawDict().data(), info.dict().GetRawDict().size(), level); } +#endif // ZSTD_ADVANCED #else // up to v0.4.x outlen = ZSTD_compress(&(*output)[output_header_len], compressBound, input, length, level); @@ -1387,17 +1444,28 @@ inline bool ZSTD_Compress(const CompressionInfo& info, const char* input, // @param compression_dict Data for presetting the compression library's // dictionary. +// @param error_message If not null, will be set if decompression fails. +// +// Returns nullptr if decompression fails. inline CacheAllocationPtr ZSTD_Uncompress( const UncompressionInfo& info, const char* input_data, size_t input_length, - size_t* uncompressed_size, MemoryAllocator* allocator = nullptr) { + size_t* uncompressed_size, MemoryAllocator* allocator = nullptr, + const char** error_message = nullptr) { #ifdef ZSTD + static const char* const kErrorDecodeOutputSize = + "Cannot decode output size."; + static const char* const kErrorOutputLenMismatch = + "Decompressed size does not match header."; uint32_t output_len = 0; if (!compression::GetDecompressedSizeInfo(&input_data, &input_length, &output_len)) { + if (error_message) { + *error_message = kErrorDecodeOutputSize; + } return nullptr; } - auto output = AllocateBlock(output_len, allocator); + CacheAllocationPtr output = AllocateBlock(output_len, allocator); size_t actual_output_length = 0; #if ZSTD_VERSION_NUMBER >= 500 // v0.5.0+ ZSTD_DCtx* context = info.context().GetZSTDContext(); @@ -1407,19 +1475,31 @@ inline CacheAllocationPtr ZSTD_Uncompress( actual_output_length = ZSTD_decompress_usingDDict( context, output.get(), output_len, input_data, input_length, info.dict().GetDigestedZstdDDict()); - } + } else { #endif // ROCKSDB_ZSTD_DDICT - if (actual_output_length == 0) { actual_output_length = ZSTD_decompress_usingDict( context, output.get(), output_len, input_data, input_length, info.dict().GetRawDict().data(), info.dict().GetRawDict().size()); +#ifdef ROCKSDB_ZSTD_DDICT } +#endif // ROCKSDB_ZSTD_DDICT #else // up to v0.4.x (void)info; actual_output_length = ZSTD_decompress(output.get(), output_len, input_data, input_length); #endif // ZSTD_VERSION_NUMBER >= 500 - assert(actual_output_length == output_len); + if (ZSTD_isError(actual_output_length)) { + if (error_message) { + *error_message = ZSTD_getErrorName(actual_output_length); + } + return nullptr; + } else if (actual_output_length != output_len) { + if (error_message) { + *error_message = kErrorOutputLenMismatch; + } + return nullptr; + } + *uncompressed_size = actual_output_length; return output; #else // ZSTD @@ -1428,6 +1508,7 @@ inline CacheAllocationPtr ZSTD_Uncompress( (void)input_length; (void)uncompressed_size; (void)allocator; + (void)error_message; return nullptr; #endif } @@ -1530,6 +1611,7 @@ inline std::string ZSTD_FinalizeDictionary( return dict_data; } #else // up to v1.4.4 + assert(false); (void)samples; (void)sample_lens; (void)max_dict_bytes; @@ -1589,7 +1671,8 @@ inline bool CompressData(const Slice& raw, inline CacheAllocationPtr UncompressData( const UncompressionInfo& uncompression_info, const char* data, size_t n, size_t* uncompressed_size, uint32_t compress_format_version, - MemoryAllocator* allocator = nullptr) { + MemoryAllocator* allocator = nullptr, + const char** error_message = nullptr) { switch (uncompression_info.type()) { case kSnappyCompression: return Snappy_Uncompress(data, n, uncompressed_size, allocator); @@ -1609,8 +1692,9 @@ inline CacheAllocationPtr UncompressData( return CacheAllocationPtr(XPRESS_Uncompress(data, n, uncompressed_size)); case kZSTD: case kZSTDNotFinalCompression: + // TODO(cbi): error message handling for other compression algorithms. return ZSTD_Uncompress(uncompression_info, data, n, uncompressed_size, - allocator); + allocator, error_message); default: return CacheAllocationPtr(); } @@ -1743,7 +1827,7 @@ class ZSTDStreamingCompress final : public StreamingCompress { size_t max_output_len) : StreamingCompress(kZSTD, opts, compress_format_version, max_output_len) { -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED cctx_ = ZSTD_createCCtx(); // Each compressed frame will have a checksum ZSTD_CCtx_setParameter(cctx_, ZSTD_c_checksumFlag, 1); @@ -1752,14 +1836,14 @@ class ZSTDStreamingCompress final : public StreamingCompress { #endif } ~ZSTDStreamingCompress() override { -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED ZSTD_freeCCtx(cctx_); #endif } int Compress(const char* input, size_t input_size, char* output, size_t* output_pos) override; void Reset() override; -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED ZSTD_CCtx* cctx_; ZSTD_inBuffer input_buffer_; #endif @@ -1770,14 +1854,14 @@ class ZSTDStreamingUncompress final : public StreamingUncompress { explicit ZSTDStreamingUncompress(uint32_t compress_format_version, size_t max_output_len) : StreamingUncompress(kZSTD, compress_format_version, max_output_len) { -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED dctx_ = ZSTD_createDCtx(); assert(dctx_ != nullptr); input_buffer_ = {/*src=*/nullptr, /*size=*/0, /*pos=*/0}; #endif } ~ZSTDStreamingUncompress() override { -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED ZSTD_freeDCtx(dctx_); #endif } @@ -1786,7 +1870,7 @@ class ZSTDStreamingUncompress final : public StreamingUncompress { void Reset() override; private: -#ifdef ZSTD_STREAMING +#ifdef ZSTD_ADVANCED ZSTD_DCtx* dctx_; ZSTD_inBuffer input_buffer_; #endif diff --git a/util/compression_context_cache.cc b/util/compression_context_cache.cc index 52c3fac72..789cc7b62 100644 --- a/util/compression_context_cache.cc +++ b/util/compression_context_cache.cc @@ -67,7 +67,7 @@ static_assert(sizeof(ZSTDCachedData) % CACHE_LINE_SIZE == 0, class CompressionContextCache::Rep { public: - Rep() {} + Rep() = default; ZSTDUncompressCachedData GetZSTDUncompressData() { auto p = per_core_uncompr_.AccessElementAndIndex(); int64_t idx = static_cast(p.second); diff --git a/util/core_local.h b/util/core_local.h index 25174aef8..9c5b3f281 100644 --- a/util/core_local.h +++ b/util/core_local.h @@ -13,6 +13,7 @@ #include "port/likely.h" #include "port/port.h" +#include "util/math.h" #include "util/random.h" namespace ROCKSDB_NAMESPACE { @@ -70,7 +71,7 @@ std::pair CoreLocalArray::AccessElementAndIndex() const { // cpu id unavailable, just pick randomly core_idx = Random::GetTLSInstance()->Uniform(1 << size_shift_); } else { - core_idx = static_cast(cpuid & ((1 << size_shift_) - 1)); + core_idx = static_cast(BottomNBits(cpuid, size_shift_)); } return {AccessAtCore(core_idx), core_idx}; } diff --git a/util/crc32c.cc b/util/crc32c.cc index d4cd78b52..38a69bb50 100644 --- a/util/crc32c.cc +++ b/util/crc32c.cc @@ -11,9 +11,8 @@ // four bytes at a time. #include "util/crc32c.h" -#include - #include +#include #include #include "port/lang.h" @@ -1117,7 +1116,13 @@ static inline Function Choose_Extend() { } #elif defined(__SSE4_2__) && defined(__PCLMUL__) && !defined NO_THREEWAY_CRC32C // NOTE: runtime detection no longer supported on x86 - (void)ExtendImpl; // suppress unused warning +#ifdef _MSC_VER +#pragma warning(disable: 4551) +#endif + (void)ExtendImpl; // suppress unused warning +#ifdef _MSC_VER +#pragma warning(default: 4551) +#endif return crc32c_3way; #else return ExtendImpl; diff --git a/util/distributed_mutex.h b/util/distributed_mutex.h index 1734269cc..e3450d753 100644 --- a/util/distributed_mutex.h +++ b/util/distributed_mutex.h @@ -28,7 +28,7 @@ class DMutex : public folly::DistributedMutex { explicit DMutex(bool IGNORED_adaptive = false) { (void)IGNORED_adaptive; } // currently no-op - void AssertHeld() {} + void AssertHeld() const {} }; using DMutexLock = std::lock_guard; diff --git a/util/dynamic_bloom.h b/util/dynamic_bloom.h index 40cd29404..0ff1053ca 100644 --- a/util/dynamic_bloom.h +++ b/util/dynamic_bloom.h @@ -126,7 +126,7 @@ inline void DynamicBloom::MayContain(int num_keys, Slice* keys, std::array byte_offsets; for (int i = 0; i < num_keys; ++i) { hashes[i] = BloomHash(keys[i]); - size_t a = FastRange32(kLen, hashes[i]); + size_t a = FastRange32(hashes[i], kLen); PREFETCH(data_ + a, 0, 3); byte_offsets[i] = a; } @@ -142,7 +142,7 @@ inline void DynamicBloom::MayContain(int num_keys, Slice* keys, #pragma warning(disable : 4189) #endif inline void DynamicBloom::Prefetch(uint32_t h32) { - size_t a = FastRange32(kLen, h32); + size_t a = FastRange32(h32, kLen); PREFETCH(data_ + a, 0, 3); } #if defined(_MSC_VER) @@ -171,7 +171,7 @@ inline void DynamicBloom::Prefetch(uint32_t h32) { // because of false positives.) inline bool DynamicBloom::MayContainHash(uint32_t h32) const { - size_t a = FastRange32(kLen, h32); + size_t a = FastRange32(h32, kLen); PREFETCH(data_ + a, 0, 3); return DoubleProbe(h32, a); } @@ -195,7 +195,7 @@ inline bool DynamicBloom::DoubleProbe(uint32_t h32, size_t byte_offset) const { template inline void DynamicBloom::AddHash(uint32_t h32, const OrFunc& or_func) { - size_t a = FastRange32(kLen, h32); + size_t a = FastRange32(h32, kLen); PREFETCH(data_ + a, 0, 3); // Expand/remix with 64-bit golden ratio uint64_t h = 0x9e3779b97f4a7c13ULL * h32; diff --git a/util/dynamic_bloom_test.cc b/util/dynamic_bloom_test.cc index 925c5479a..949ab8f76 100644 --- a/util/dynamic_bloom_test.cc +++ b/util/dynamic_bloom_test.cc @@ -164,10 +164,11 @@ TEST_F(DynamicBloomTest, VaryingLengths) { "%5.2f%% @ num = %6u, bloom_bits = %6u\n", nonseq ? "nonseq" : "seq", rate * 100.0, num, bloom_bits); - if (rate > 0.0125) + if (rate > 0.0125) { mediocre_filters++; // Allowed, but not too often - else + } else { good_filters++; + } } } diff --git a/util/file_checksum_helper.cc b/util/file_checksum_helper.cc index b8c4099b8..59da96fa8 100644 --- a/util/file_checksum_helper.cc +++ b/util/file_checksum_helper.cc @@ -31,7 +31,7 @@ Status FileChecksumListImpl::GetAllFileChecksums( return Status::InvalidArgument("Pointer has not been initiated"); } - for (auto i : checksum_map_) { + for (const auto& i : checksum_map_) { file_numbers->push_back(i.first); checksums->push_back(i.second.first); checksum_func_names->push_back(i.second.second); diff --git a/util/file_reader_writer_test.cc b/util/file_reader_writer_test.cc index 68776612b..036c030dc 100644 --- a/util/file_reader_writer_test.cc +++ b/util/file_reader_writer_test.cc @@ -31,7 +31,7 @@ TEST_F(WritableFileWriterTest, RangeSync) { class FakeWF : public FSWritableFile { public: explicit FakeWF() : size_(0), last_synced_(0) {} - ~FakeWF() override {} + ~FakeWF() override = default; using FSWritableFile::Append; IOStatus Append(const Slice& data, const IOOptions& /*options*/, @@ -134,7 +134,7 @@ TEST_F(WritableFileWriterTest, IncrementalBuffer) { : file_data_(_file_data), use_direct_io_(_use_direct_io), no_flush_(_no_flush) {} - ~FakeWF() override {} + ~FakeWF() override = default; using FSWritableFile::Append; IOStatus Append(const Slice& data, const IOOptions& /*options*/, @@ -588,7 +588,7 @@ class ReadaheadSequentialFileTest : public testing::Test, scratch_.reset(new char[2 * readahead_size_]); ResetSourceStr(); } - ReadaheadSequentialFileTest() {} + ReadaheadSequentialFileTest() = default; std::string Read(size_t n) { Slice result; Status s = test_read_holder_->Read( @@ -919,7 +919,7 @@ class WritableFileWriterIOPriorityTest : public testing::Test { class FakeWF : public FSWritableFile { public: explicit FakeWF(Env::IOPriority io_priority) { SetIOPriority(io_priority); } - ~FakeWF() override {} + ~FakeWF() override = default; IOStatus Append(const Slice& /*data*/, const IOOptions& options, IODebugContext* /*dbg*/) override { diff --git a/util/filelock_test.cc b/util/filelock_test.cc index 82021aec9..7a41449bf 100644 --- a/util/filelock_test.cc +++ b/util/filelock_test.cc @@ -34,7 +34,7 @@ class LockTest : public testing::Test { current_ = this; } - ~LockTest() override {} + ~LockTest() override = default; Status LockFile(FileLock** db_lock) { return env_->LockFile(file_, db_lock); } @@ -94,8 +94,9 @@ class LockTest : public testing::Test { } else if (pid > 0) { // parent process int status; - while (-1 == waitpid(pid, &status, 0)) + while (-1 == waitpid(pid, &status, 0)) { ; + } if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { // child process exited with non success status return false; diff --git a/util/hash_test.cc b/util/hash_test.cc index 72112b044..ccc283a24 100644 --- a/util/hash_test.cc +++ b/util/hash_test.cc @@ -565,6 +565,8 @@ size_t FastRange64(uint64_t hash, size_t range) { // Tests for math.h / math128.h (not worth a separate test binary) using ROCKSDB_NAMESPACE::BitParity; using ROCKSDB_NAMESPACE::BitsSetToOne; +using ROCKSDB_NAMESPACE::BitwiseAnd; +using ROCKSDB_NAMESPACE::BottomNBits; using ROCKSDB_NAMESPACE::ConstexprFloorLog2; using ROCKSDB_NAMESPACE::CountTrailingZeroBits; using ROCKSDB_NAMESPACE::DecodeFixed128; @@ -580,6 +582,19 @@ using ROCKSDB_NAMESPACE::Upper64of128; int blah(int x) { return DownwardInvolution(x); } +template +static void test_BitwiseAnd(T1 v1, T2 v2) { + auto a = BitwiseAnd(v1, v2); + // Essentially repeating the implementation :-/ + if constexpr (sizeof(T1) < sizeof(T2)) { + static_assert(std::is_same_v); + EXPECT_EQ(a, static_cast(v1 & v2)); + } else { + static_assert(std::is_same_v); + EXPECT_EQ(a, static_cast(v1 & v2)); + } +} + template static void test_BitOps() { // This complex code is to generalize to 128-bit values. Otherwise @@ -598,6 +613,22 @@ static void test_BitOps() { // If we could directly use arithmetic: // T vm1 = static_cast(v - 1); + // BottomNBits + { + // An essentially full length value + T x = everyOtherBit; + if (i > 2) { + // Make it slightly irregular + x = x ^ (T{1} << (i / 2)); + } + auto a = BottomNBits(x, i); + auto b = BottomNBits(~x, i); + EXPECT_EQ(x | a, x); + EXPECT_EQ(a | b, vm1); + EXPECT_EQ(a & b, T{0}); + EXPECT_EQ(BottomNBits(x ^ a, i), T{0}); + } + // FloorLog2 if (v > 0) { EXPECT_EQ(FloorLog2(v), i); @@ -707,9 +738,22 @@ static void test_BitOps() { } } + // BitwiseAnd + { + test_BitwiseAnd(vm1, static_cast(0x99)); + test_BitwiseAnd(v, static_cast(0x99)); + test_BitwiseAnd(char{0x66}, vm1); + test_BitwiseAnd(char{0x66}, v); + test_BitwiseAnd(v, int16_t{0x6699}); + test_BitwiseAnd(v, uint16_t{0x9966}); + test_BitwiseAnd(int64_t{0x1234234534564567}, v); + test_BitwiseAnd(uint64_t{0x9876876576545432}, v); + } + vm1 = (vm1 << 1) | 1; } + // ConstexprFloorLog2 EXPECT_EQ(ConstexprFloorLog2(T{1}), 0); EXPECT_EQ(ConstexprFloorLog2(T{2}), 1); EXPECT_EQ(ConstexprFloorLog2(T{3}), 1); diff --git a/util/math.h b/util/math.h index 39f308328..e1948e0a3 100644 --- a/util/math.h +++ b/util/math.h @@ -9,6 +9,9 @@ #ifdef _MSC_VER #include #endif +#ifdef __BMI2__ +#include +#endif #include #include @@ -20,11 +23,33 @@ ASSERT_FEATURE_COMPAT_HEADER(); namespace ROCKSDB_NAMESPACE { +// Fast implementation of extracting the bottom n bits of an integer. +// To ensure fast implementation, undefined if n bits is full width or more. +template +inline T BottomNBits(T v, int nbits) { + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); + assert(nbits >= 0); + assert(nbits < int{8 * sizeof(T)}); +#ifdef __BMI2__ + if constexpr (sizeof(T) <= 4) { + return static_cast(_bzhi_u32(static_cast(v), nbits)); + } + if constexpr (sizeof(T) <= 8) { + return static_cast(_bzhi_u64(static_cast(v), nbits)); + } +#endif + // Newer compilers compile this down to bzhi on x86, but some older + // ones don't, thus the need for the intrinsic above. + return static_cast(v & ((T{1} << nbits) - 1)); +} + // Fast implementation of floor(log2(v)). Undefined for 0 or negative // numbers (in case of signed type). template inline int FloorLog2(T v) { - static_assert(std::is_integral::value, "non-integral type"); + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); assert(v > 0); #ifdef _MSC_VER static_assert(sizeof(T) <= sizeof(uint64_t), "type too big"); @@ -63,6 +88,8 @@ inline int FloorLog2(T v) { // Constexpr version of FloorLog2 template constexpr int ConstexprFloorLog2(T v) { + // NOTE: not checking is_integral so that this works with Unsigned128 + static_assert(!std::is_reference_v, "use std::remove_reference_t"); int rv = 0; while (v > T{1}) { ++rv; @@ -74,7 +101,8 @@ constexpr int ConstexprFloorLog2(T v) { // Number of low-order zero bits before the first 1 bit. Undefined for 0. template inline int CountTrailingZeroBits(T v) { - static_assert(std::is_integral::value, "non-integral type"); + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); assert(v != 0); #ifdef _MSC_VER static_assert(sizeof(T) <= sizeof(uint64_t), "type too big"); @@ -115,6 +143,9 @@ namespace detail { template int BitsSetToOneFallback(T v) { + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); + const int kBits = static_cast(sizeof(T)) * 8; static_assert((kBits & (kBits - 1)) == 0, "must be power of two bits"); // we static_cast these bit patterns in order to truncate them to the correct @@ -140,7 +171,9 @@ int BitsSetToOneFallback(T v) { // Number of bits set to 1. Also known as "population count". template inline int BitsSetToOne(T v) { - static_assert(std::is_integral::value, "non-integral type"); + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); + #ifdef _MSC_VER static_assert(sizeof(T) <= sizeof(uint64_t), "type too big"); if (sizeof(T) < sizeof(uint32_t)) { @@ -192,7 +225,9 @@ inline int BitsSetToOne(T v) { template inline int BitParity(T v) { - static_assert(std::is_integral::value, "non-integral type"); + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); + #ifdef _MSC_VER // bit parity == oddness of popcount return BitsSetToOne(v) & 1; @@ -214,7 +249,8 @@ inline int BitParity(T v) { // encode/decode big endian. template inline T EndianSwapValue(T v) { - static_assert(std::is_integral::value, "non-integral type"); + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); #ifdef _MSC_VER if (sizeof(T) == 2) { @@ -244,6 +280,9 @@ inline T EndianSwapValue(T v) { // Reverses the order of bits in an integral value template inline T ReverseBits(T v) { + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); + T r = EndianSwapValue(v); const T kHighestByte = T{1} << ((sizeof(T) - 1) * 8); const T kEveryByte = kHighestByte | (kHighestByte / 255); @@ -277,7 +316,8 @@ inline T ReverseBits(T v) { // is that all square sub-matrices that include the top row are invertible. template inline T DownwardInvolution(T v) { - static_assert(std::is_integral::value, "non-integral type"); + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); static_assert(sizeof(T) <= 8, "only supported up to 64 bits"); uint64_t r = static_cast(v); @@ -296,4 +336,16 @@ inline T DownwardInvolution(T v) { return static_cast(r); } +// Bitwise-And with typing that allows you to avoid writing an explicit cast +// to the smaller type, or the type of the right parameter if same size. +template +inline std::conditional_t BitwiseAnd(A a, B b) { + static_assert(std::is_integral_v, "non-integral type"); + static_assert(std::is_integral_v, "non-integral type"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); + static_assert(!std::is_reference_v, "use std::remove_reference_t"); + using Smaller = std::conditional_t; + return static_cast(a & b); +} + } // namespace ROCKSDB_NAMESPACE diff --git a/util/math128.h b/util/math128.h index ae490051a..5f96dbc66 100644 --- a/util/math128.h +++ b/util/math128.h @@ -41,13 +41,13 @@ struct Unsigned128 { hi = upper; } - explicit operator uint64_t() { return lo; } - - explicit operator uint32_t() { return static_cast(lo); } - - explicit operator uint16_t() { return static_cast(lo); } - - explicit operator uint8_t() { return static_cast(lo); } + // Convert to any integer 64 bits or less. + template && + sizeof(T) <= sizeof(uint64_t)> > + explicit operator T() { + return static_cast(lo); + } }; inline Unsigned128 operator<<(const Unsigned128& lhs, unsigned shift) { @@ -190,6 +190,16 @@ inline Unsigned128 Multiply64to128(uint64_t a, uint64_t b) { #endif } +template <> +inline Unsigned128 BottomNBits(Unsigned128 v, int nbits) { + if (nbits < 64) { + return BottomNBits(Lower64of128(v), nbits); + } else { + return (Unsigned128{BottomNBits(Upper64of128(v), nbits - 64)} << 64) | + Lower64of128(v); + } +} + template <> inline int FloorLog2(Unsigned128 v) { if (Upper64of128(v) == 0) { @@ -236,6 +246,18 @@ inline Unsigned128 DownwardInvolution(Unsigned128 v) { DownwardInvolution(Upper64of128(v) ^ Lower64of128(v)); } +template +inline std::remove_reference_t BitwiseAnd(A a, Unsigned128 b) { + static_assert(sizeof(A) <= sizeof(Unsigned128)); + return static_cast(a & b); +} + +template +inline std::remove_reference_t BitwiseAnd(Unsigned128 a, B b) { + static_assert(sizeof(B) <= sizeof(Unsigned128)); + return static_cast(a & b); +} + template struct IsUnsignedUpTo128 : std::integral_constant::value || diff --git a/util/overload.h b/util/overload.h new file mode 100644 index 000000000..27da81648 --- /dev/null +++ b/util/overload.h @@ -0,0 +1,23 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// A helper template that can combine multiple functors into a single one to be +// used with std::visit for example. It also works with lambdas, since it +// comes with an explicit deduction guide. +template +struct overload : Ts... { + using Ts::operator()...; +}; + +template +overload(Ts...) -> overload; + +} // namespace ROCKSDB_NAMESPACE diff --git a/util/random.cc b/util/random.cc index 7ac6ee19a..17396d32f 100644 --- a/util/random.cc +++ b/util/random.cc @@ -6,10 +6,9 @@ #include "util/random.h" -#include -#include -#include - +#include +#include +#include #include #include diff --git a/util/rate_limiter.cc b/util/rate_limiter.cc index ddb9bdbf0..e92b3bf76 100644 --- a/util/rate_limiter.cc +++ b/util/rate_limiter.cc @@ -107,6 +107,23 @@ void GenericRateLimiter::SetBytesPerSecondLocked(int64_t bytes_per_second) { std::memory_order_relaxed); } +Status GenericRateLimiter::SetSingleBurstBytes(int64_t single_burst_bytes) { + if (single_burst_bytes <= 0) { + return Status::InvalidArgument( + "`single_burst_bytes` must be greater than 0"); + } + + MutexLock g(&request_mutex_); + SetSingleBurstBytesLocked(single_burst_bytes); + return Status::OK(); +} + +void GenericRateLimiter::SetSingleBurstBytesLocked(int64_t single_burst_bytes) { + refill_bytes_per_period_.store(single_burst_bytes, std::memory_order_relaxed); + refill_period_us_.store(CalculateRefillPeriodUsLocked(single_burst_bytes), + std::memory_order_relaxed); +} + void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri, Statistics* stats) { assert(bytes <= refill_bytes_per_period_.load(std::memory_order_relaxed)); @@ -120,7 +137,8 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri, static const int kRefillsPerTune = 100; std::chrono::microseconds now(NowMicrosMonotonicLocked()); if (now - tuned_time_ >= - kRefillsPerTune * std::chrono::microseconds(refill_period_us_)) { + kRefillsPerTune * std::chrono::microseconds(refill_period_us_.load( + std::memory_order_relaxed))) { Status s = TuneLocked(); s.PermitUncheckedError(); //**TODO: What to do on error? } @@ -170,7 +188,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri, RecordTick(stats, NUMBER_RATE_LIMITER_DRAINS); ++num_drains_; wait_until_refill_pending_ = true; - r.cv.TimedWait(wait_until); + clock_->TimedWait(&r.cv, std::chrono::microseconds(wait_until)); TEST_SYNC_POINT_CALLBACK("GenericRateLimiter::Request:PostTimedWait", &time_until_refill_us); wait_until_refill_pending_ = false; @@ -261,7 +279,8 @@ GenericRateLimiter::GeneratePriorityIterationOrderLocked() { void GenericRateLimiter::RefillBytesAndGrantRequestsLocked() { TEST_SYNC_POINT_CALLBACK( "GenericRateLimiter::RefillBytesAndGrantRequestsLocked", &request_mutex_); - next_refill_us_ = NowMicrosMonotonicLocked() + refill_period_us_; + next_refill_us_ = NowMicrosMonotonicLocked() + + refill_period_us_.load(std::memory_order_relaxed); // Carry over the left over quota from the last period auto refill_bytes_per_period = refill_bytes_per_period_.load(std::memory_order_relaxed); @@ -299,13 +318,28 @@ void GenericRateLimiter::RefillBytesAndGrantRequestsLocked() { int64_t GenericRateLimiter::CalculateRefillBytesPerPeriodLocked( int64_t rate_bytes_per_sec) { + int64_t refill_period_us = refill_period_us_.load(std::memory_order_relaxed); if (std::numeric_limits::max() / rate_bytes_per_sec < - refill_period_us_) { + refill_period_us) { + // Avoid unexpected result in the overflow case. The result now is still + // inaccurate but is a number that is large enough. + return std::numeric_limits::max() / kMicrosecondsPerSecond; + } else { + return rate_bytes_per_sec * refill_period_us / kMicrosecondsPerSecond; + } +} + +int64_t GenericRateLimiter::CalculateRefillPeriodUsLocked( + int64_t single_burst_bytes) { + int64_t rate_bytes_per_sec = + rate_bytes_per_sec_.load(std::memory_order_relaxed); + if (std::numeric_limits::max() / single_burst_bytes < + kMicrosecondsPerSecond) { // Avoid unexpected result in the overflow case. The result now is still // inaccurate but is a number that is large enough. - return std::numeric_limits::max() / 1000000; + return std::numeric_limits::max() / rate_bytes_per_sec; } else { - return rate_bytes_per_sec * refill_period_us_ / 1000000; + return single_burst_bytes * kMicrosecondsPerSecond / rate_bytes_per_sec; } } @@ -320,10 +354,11 @@ Status GenericRateLimiter::TuneLocked() { std::chrono::microseconds prev_tuned_time = tuned_time_; tuned_time_ = std::chrono::microseconds(NowMicrosMonotonicLocked()); + int64_t refill_period_us = refill_period_us_.load(std::memory_order_relaxed); int64_t elapsed_intervals = (tuned_time_ - prev_tuned_time + - std::chrono::microseconds(refill_period_us_) - + std::chrono::microseconds(refill_period_us) - std::chrono::microseconds(1)) / - std::chrono::microseconds(refill_period_us_); + std::chrono::microseconds(refill_period_us); // We tune every kRefillsPerTune intervals, so the overflow and division-by- // zero conditions should never happen. assert(num_drains_ <= std::numeric_limits::max() / 100); diff --git a/util/rate_limiter_impl.h b/util/rate_limiter_impl.h index 4c078f5a0..c6786b048 100644 --- a/util/rate_limiter_impl.h +++ b/util/rate_limiter_impl.h @@ -36,6 +36,8 @@ class GenericRateLimiter : public RateLimiter { // This API allows user to dynamically change rate limiter's bytes per second. virtual void SetBytesPerSecond(int64_t bytes_per_second) override; + virtual Status SetSingleBurstBytes(int64_t single_burst_bytes) override; + // Request for token to write bytes. If this request can not be satisfied, // the call is blocked. Caller is responsible to make sure // bytes <= GetSingleBurstBytes() and bytes >= 0. Negative bytes @@ -102,11 +104,14 @@ class GenericRateLimiter : public RateLimiter { } private: + static constexpr int kMicrosecondsPerSecond = 1000000; void RefillBytesAndGrantRequestsLocked(); std::vector GeneratePriorityIterationOrderLocked(); int64_t CalculateRefillBytesPerPeriodLocked(int64_t rate_bytes_per_sec); + int64_t CalculateRefillPeriodUsLocked(int64_t single_burst_bytes); Status TuneLocked(); void SetBytesPerSecondLocked(int64_t bytes_per_second); + void SetSingleBurstBytesLocked(int64_t single_burst_bytes); uint64_t NowMicrosMonotonicLocked() { return clock_->NowNanos() / std::milli::den; @@ -115,7 +120,7 @@ class GenericRateLimiter : public RateLimiter { // This mutex guard all internal states mutable port::Mutex request_mutex_; - const int64_t refill_period_us_; + std::atomic refill_period_us_; std::atomic rate_bytes_per_sec_; std::atomic refill_bytes_per_period_; diff --git a/util/rate_limiter_test.cc b/util/rate_limiter_test.cc index 7df2bb04f..16e7623ac 100644 --- a/util/rate_limiter_test.cc +++ b/util/rate_limiter_test.cc @@ -15,6 +15,7 @@ #include "db/db_test_util.h" #include "port/port.h" #include "rocksdb/system_clock.h" +#include "test_util/mock_time_env.h" #include "test_util/sync_point.h" #include "test_util/testharness.h" #include "util/random.h" @@ -464,31 +465,21 @@ TEST_F(RateLimiterTest, AutoTuneIncreaseWhenFull) { const std::chrono::seconds kTimePerRefill(1); const int kRefillsPerTune = 100; // needs to match util/rate_limiter.cc - SpecialEnv special_env(Env::Default(), /*time_elapse_only_sleep*/ true); + auto mock_clock = + std::make_shared(Env::Default()->GetSystemClock()); auto stats = CreateDBStatistics(); std::unique_ptr rate_limiter(new GenericRateLimiter( 1000 /* rate_bytes_per_sec */, std::chrono::microseconds(kTimePerRefill).count(), 10 /* fairness */, - RateLimiter::Mode::kWritesOnly, special_env.GetSystemClock(), - true /* auto_tuned */)); - - // Rate limiter uses `CondVar::TimedWait()`, which does not have access to the - // `Env` to advance its time according to the fake wait duration. The - // workaround is to install a callback that advance the `Env`'s mock time. - ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( - "GenericRateLimiter::Request:PostTimedWait", [&](void* arg) { - int64_t time_waited_us = *static_cast(arg); - special_env.SleepForMicroseconds(static_cast(time_waited_us)); - }); - ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + RateLimiter::Mode::kWritesOnly, mock_clock, true /* auto_tuned */)); // verify rate limit increases after a sequence of periods where rate limiter // is always drained int64_t orig_bytes_per_sec = rate_limiter->GetSingleBurstBytes(); rate_limiter->Request(orig_bytes_per_sec, Env::IO_HIGH, stats.get(), RateLimiter::OpType::kWrite); - while (std::chrono::microseconds(special_env.NowMicros()) <= + while (std::chrono::microseconds(mock_clock->NowMicros()) <= kRefillsPerTune * kTimePerRefill) { rate_limiter->Request(orig_bytes_per_sec, Env::IO_HIGH, stats.get(), RateLimiter::OpType::kWrite); @@ -496,13 +487,9 @@ TEST_F(RateLimiterTest, AutoTuneIncreaseWhenFull) { int64_t new_bytes_per_sec = rate_limiter->GetSingleBurstBytes(); ASSERT_GT(new_bytes_per_sec, orig_bytes_per_sec); - ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); - ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( - "GenericRateLimiter::Request:PostTimedWait"); - // decreases after a sequence of periods where rate limiter is not drained orig_bytes_per_sec = new_bytes_per_sec; - special_env.SleepForMicroseconds(static_cast( + mock_clock->SleepForMicroseconds(static_cast( kRefillsPerTune * std::chrono::microseconds(kTimePerRefill).count())); // make a request so tuner can be triggered rate_limiter->Request(1 /* bytes */, Env::IO_HIGH, stats.get(), @@ -511,6 +498,90 @@ TEST_F(RateLimiterTest, AutoTuneIncreaseWhenFull) { ASSERT_LT(new_bytes_per_sec, orig_bytes_per_sec); } +TEST_F(RateLimiterTest, WaitHangingBug) { + // At t=0: Threads 0 and 1 request `kBytesPerRefill` bytes at low-pri. One + // will be granted immediately and the other will enter `TimedWait()`. + // + // At t=`kMicrosPerRefill`: Thread 2 requests `kBytesPerRefill` bytes at + // low-pri. Thread 2's request enters the queue. To expose the bug scenario, + // `SyncPoint`s ensure this happens while the lock is temporarily released in + // `TimedWait()`. Before the bug fix, Thread 2's request would then hang in + // `Wait()` interminably. + const int kBytesPerSecond = 100; + const int kMicrosPerSecond = 1000 * 1000; + const int kMicrosPerRefill = kMicrosPerSecond; + const int kBytesPerRefill = + kBytesPerSecond * kMicrosPerRefill / kMicrosPerSecond; + + auto mock_clock = + std::make_shared(Env::Default()->GetSystemClock()); + std::unique_ptr limiter(new GenericRateLimiter( + kBytesPerSecond, kMicrosPerRefill, 10 /* fairness */, + RateLimiter::Mode::kWritesOnly, mock_clock, false /* auto_tuned */)); + std::array request_threads; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"RateLimiterTest::WaitHangingBug:InitialRequestsReady", + "MockSystemClock::TimedWait:UnlockedPreSleep"}, + {"MockSystemClock::TimedWait:UnlockedPostSleep1", + "RateLimiterTest::WaitHangingBug:TestThreadRequestBegin"}, + {"RateLimiterTest::WaitHangingBug:TestThreadRequestEnd", + "MockSystemClock::TimedWait:UnlockedPostSleep2"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 0; i < 2; i++) { + request_threads[i] = std::thread([&]() { + limiter->Request(kBytesPerRefill /* bytes */, Env::IOPriority::IO_LOW, + nullptr /* stats */, RateLimiter::OpType::kWrite); + }); + } + while (limiter->GetTotalRequests() < 2) { + } + TEST_SYNC_POINT("RateLimiterTest::WaitHangingBug:InitialRequestsReady"); + + TEST_SYNC_POINT("RateLimiterTest::WaitHangingBug:TestThreadRequestBegin"); + request_threads[2] = std::thread([&]() { + limiter->Request(kBytesPerRefill /* bytes */, Env::IOPriority::IO_LOW, + nullptr /* stats */, RateLimiter::OpType::kWrite); + }); + while (limiter->GetTotalRequests() < 3) { + } + TEST_SYNC_POINT("RateLimiterTest::WaitHangingBug:TestThreadRequestEnd"); + + for (int i = 0; i < 3; i++) { + request_threads[i].join(); + } +} + +TEST_F(RateLimiterTest, RuntimeSingleBurstBytesChange) { + constexpr int kMicrosecondsPerSecond = 1000000; + + const int64_t kRateBytesPerSec = 400; + + const int64_t kOldSingleBurstBytes = 100; + const int64_t kOldRefillPeriodUs = + kOldSingleBurstBytes * kMicrosecondsPerSecond / kRateBytesPerSec; + const int64_t kNewSingleBurstBytes = kOldSingleBurstBytes * 2; + + SpecialEnv special_env(Env::Default(), /*time_elapse_only_sleep*/ true); + std::unique_ptr limiter(new GenericRateLimiter( + kRateBytesPerSec, kOldRefillPeriodUs, 10 /* fairness */, + RateLimiter::Mode::kWritesOnly, special_env.GetSystemClock(), + false /* auto_tuned */)); + + ASSERT_EQ(kOldSingleBurstBytes, limiter->GetSingleBurstBytes()); + + ASSERT_TRUE(limiter->SetSingleBurstBytes(0).IsInvalidArgument()); + ASSERT_OK(limiter->SetSingleBurstBytes(kNewSingleBurstBytes)); + ASSERT_EQ(kNewSingleBurstBytes, limiter->GetSingleBurstBytes()); + + // If the updated single burst bytes is not reflected in the bytes + // granting process, this request will hang forever. + limiter->Request(limiter->GetSingleBurstBytes() /* bytes */, + Env::IOPriority::IO_USER, nullptr /* stats */, + RateLimiter::OpType::kWrite); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/util/slice_test.cc b/util/slice_test.cc index 010ded3d8..e82547494 100644 --- a/util/slice_test.cc +++ b/util/slice_test.cc @@ -243,6 +243,36 @@ TEST_F(SmallEnumSetTest, SmallEnumSetTest2) { } } +// ***************************************************************** // +// Unit test for Status +TEST(StatusTest, Update) { + const Status ok = Status::OK(); + const Status inc = Status::Incomplete("blah"); + const Status notf = Status::NotFound("meow"); + + Status s = ok; + ASSERT_TRUE(s.UpdateIfOk(Status::Corruption("bad")).IsCorruption()); + ASSERT_TRUE(s.IsCorruption()); + + s = ok; + ASSERT_TRUE(s.UpdateIfOk(Status::OK()).ok()); + ASSERT_TRUE(s.UpdateIfOk(ok).ok()); + ASSERT_TRUE(s.ok()); + + ASSERT_TRUE(s.UpdateIfOk(inc).IsIncomplete()); + ASSERT_TRUE(s.IsIncomplete()); + + ASSERT_TRUE(s.UpdateIfOk(notf).IsIncomplete()); + ASSERT_TRUE(s.UpdateIfOk(ok).IsIncomplete()); + ASSERT_TRUE(s.IsIncomplete()); + + // Keeps left-most non-OK status + s = ok; + ASSERT_TRUE( + s.UpdateIfOk(Status()).UpdateIfOk(notf).UpdateIfOk(inc).IsNotFound()); + ASSERT_TRUE(s.IsNotFound()); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/util/status.cc b/util/status.cc index ead315848..160755d54 100644 --- a/util/status.cc +++ b/util/status.cc @@ -45,6 +45,7 @@ static const char* msgs[static_cast(Status::kMaxSubCode)] = { "Txn not prepared", // kTxnNotPrepared "IO fenced off", // kIOFenced "Merge operator failed", // kMergeOperatorFailed + "Number of operands merged exceeded threshold", // kMergeOperandThresholdExceeded }; Status::Status(Code _code, SubCode _subcode, const Slice& msg, diff --git a/util/string_util.cc b/util/string_util.cc index 821ccba07..57207889f 100644 --- a/util/string_util.cc +++ b/util/string_util.cc @@ -437,6 +437,45 @@ bool SerializeIntVector(const std::vector& vec, std::string* value) { return true; } +int ParseTimeStringToSeconds(const std::string& value) { + int hours, minutes; + char colon; + + std::istringstream stream(value); + stream >> hours >> colon >> minutes; + + if (stream.fail() || !stream.eof() || colon != ':') { + return -1; + } + + if (hours < 0 || hours > 23 || minutes < 0 || minutes > 59) { + return -1; + } + return hours * 3600 + minutes * 60; +} + +bool TryParseTimeRangeString(const std::string& value, int& start_time, + int& end_time) { + if (value.empty()) { + start_time = 0; + end_time = 0; + return true; + } + auto split = StringSplit(value, '-'); + if (split.size() != 2) { + return false; + } + start_time = ParseTimeStringToSeconds(split[0]); + if (start_time < 0) { + return false; + } + end_time = ParseTimeStringToSeconds(split[1]); + if (end_time < 0) { + return false; + } + return true; +} + // Copied from folly/string.cpp: // https://github.com/facebook/folly/blob/0deef031cb8aab76dc7e736f8b7c22d701d5f36b/folly/String.cpp#L457 // There are two variants of `strerror_r` function, one returns diff --git a/util/string_util.h b/util/string_util.h index 0b15181f5..999081ebb 100644 --- a/util/string_util.h +++ b/util/string_util.h @@ -166,6 +166,16 @@ std::vector ParseVectorInt(const std::string& value); bool SerializeIntVector(const std::vector& vec, std::string* value); +// Expects HH:mm format for the input value +// Returns -1 if invalid input. Otherwise returns seconds since midnight +int ParseTimeStringToSeconds(const std::string& value); + +// Expects HH:mm-HH:mm format for the input value +// Returns false, if invalid format. +// Otherwise, returns true and start_time and end_time are set +bool TryParseTimeRangeString(const std::string& value, int& start_time, + int& end_time); + extern const std::string kNullptrString; // errnoStr() function returns a string that describes the error code passed in diff --git a/util/thread_operation.h b/util/thread_operation.h index b6c106279..4c01782ca 100644 --- a/util/thread_operation.h +++ b/util/thread_operation.h @@ -39,7 +39,16 @@ static OperationInfo global_operation_table[] = { {ThreadStatus::OP_UNKNOWN, ""}, {ThreadStatus::OP_COMPACTION, "Compaction"}, {ThreadStatus::OP_FLUSH, "Flush"}, - {ThreadStatus::OP_DBOPEN, "DBOpen"}}; + {ThreadStatus::OP_DBOPEN, "DBOpen"}, + {ThreadStatus::OP_GET, "Get"}, + {ThreadStatus::OP_MULTIGET, "MultiGet"}, + {ThreadStatus::OP_DBITERATOR, "DBIterator"}, + {ThreadStatus::OP_VERIFY_DB_CHECKSUM, "VerifyDBChecksum"}, + {ThreadStatus::OP_VERIFY_FILE_CHECKSUMS, "VerifyFileChecksums"}, + {ThreadStatus::OP_GETENTITY, "GetEntity"}, + {ThreadStatus::OP_MULTIGETENTITY, "MultiGetEntity"}, + +}; struct OperationStageInfo { const ThreadStatus::OperationStage stage; diff --git a/util/udt_util.cc b/util/udt_util.cc index 39422fa96..40cf1e496 100644 --- a/util/udt_util.cc +++ b/util/udt_util.cc @@ -8,6 +8,7 @@ #include "db/dbformat.h" #include "rocksdb/types.h" +#include "util/coding.h" #include "util/write_batch_util.h" namespace ROCKSDB_NAMESPACE { @@ -100,6 +101,40 @@ Status CheckWriteBatchTimestampSizeConsistency( } return Status::OK(); } + +enum class ToggleUDT { + kUnchanged, + kEnableUDT, + kDisableUDT, + kInvalidChange, +}; + +ToggleUDT CompareComparator(const Comparator* new_comparator, + const std::string& old_comparator_name) { + static const char* kUDTSuffix = ".u64ts"; + static const Slice kSuffixSlice = kUDTSuffix; + static const size_t kSuffixSize = 6; + size_t ts_sz = new_comparator->timestamp_size(); + (void)ts_sz; + Slice new_ucmp_name(new_comparator->Name()); + Slice old_ucmp_name(old_comparator_name); + if (new_ucmp_name.compare(old_ucmp_name) == 0) { + return ToggleUDT::kUnchanged; + } + if (new_ucmp_name.size() == old_ucmp_name.size() + kSuffixSize && + new_ucmp_name.starts_with(old_ucmp_name) && + new_ucmp_name.ends_with(kSuffixSlice)) { + assert(ts_sz == 8); + return ToggleUDT::kEnableUDT; + } + if (old_ucmp_name.size() == new_ucmp_name.size() + kSuffixSize && + old_ucmp_name.starts_with(new_ucmp_name) && + old_ucmp_name.ends_with(kSuffixSlice)) { + assert(ts_sz == 0); + return ToggleUDT::kDisableUDT; + } + return ToggleUDT::kInvalidChange; +} } // namespace TimestampRecoveryHandler::TimestampRecoveryHandler( @@ -261,4 +296,90 @@ Status HandleWriteBatchTimestampSizeDifference( } return Status::OK(); } + +Status ValidateUserDefinedTimestampsOptions( + const Comparator* new_comparator, const std::string& old_comparator_name, + bool new_persist_udt, bool old_persist_udt, + bool* mark_sst_files_has_no_udt) { + size_t ts_sz = new_comparator->timestamp_size(); + ToggleUDT res = CompareComparator(new_comparator, old_comparator_name); + switch (res) { + case ToggleUDT::kUnchanged: + if (old_persist_udt == new_persist_udt) { + return Status::OK(); + } + if (ts_sz == 0) { + return Status::OK(); + } + return Status::InvalidArgument( + "Cannot toggle the persist_user_defined_timestamps flag for a column " + "family with user-defined timestamps feature enabled."); + case ToggleUDT::kEnableUDT: + if (!new_persist_udt) { + *mark_sst_files_has_no_udt = true; + return Status::OK(); + } + return Status::InvalidArgument( + "Cannot open a column family and enable user-defined timestamps " + "feature without setting persist_user_defined_timestamps flag to " + "false."); + case ToggleUDT::kDisableUDT: + if (!old_persist_udt) { + return Status::OK(); + } + return Status::InvalidArgument( + "Cannot open a column family and disable user-defined timestamps " + "feature if its existing persist_user_defined_timestamps flag is not " + "false."); + case ToggleUDT::kInvalidChange: + return Status::InvalidArgument( + new_comparator->Name(), + "does not match existing comparator " + old_comparator_name); + default: + break; + } + return Status::InvalidArgument( + "Unsupported user defined timestamps settings change."); +} + +void GetFullHistoryTsLowFromU64CutoffTs(Slice* cutoff_ts, + std::string* full_history_ts_low) { + uint64_t cutoff_udt_ts = 0; + [[maybe_unused]] bool format_res = GetFixed64(cutoff_ts, &cutoff_udt_ts); + assert(format_res); + PutFixed64(full_history_ts_low, cutoff_udt_ts + 1); +} + +std::tuple, std::optional> +MaybeAddTimestampsToRange(const Slice* start, const Slice* end, size_t ts_sz, + std::string* start_with_ts, std::string* end_with_ts, + bool exclusive_end) { + std::optional ret_start, ret_end; + if (start) { + if (ts_sz == 0) { + ret_start = *start; + } else { + // Maximum timestamp means including all keys with any timestamp for start + AppendKeyWithMaxTimestamp(start_with_ts, *start, ts_sz); + ret_start = Slice(*start_with_ts); + } + } + if (end) { + if (ts_sz == 0) { + ret_end = *end; + } else { + if (exclusive_end) { + // Append a maximum timestamp as the range limit is exclusive: + // [start, end) + AppendKeyWithMaxTimestamp(end_with_ts, *end, ts_sz); + } else { + // Append a minimum timestamp to end so the range limit is inclusive: + // [start, end] + AppendKeyWithMinTimestamp(end_with_ts, *end, ts_sz); + } + ret_end = Slice(*end_with_ts); + } + } + return std::make_tuple(ret_start, ret_end); +} } // namespace ROCKSDB_NAMESPACE diff --git a/util/udt_util.h b/util/udt_util.h index fc980a04b..b524fceab 100644 --- a/util/udt_util.h +++ b/util/udt_util.h @@ -215,4 +215,54 @@ Status HandleWriteBatchTimestampSizeDifference( const UnorderedMap& record_ts_sz, TimestampSizeConsistencyMode check_mode, std::unique_ptr* new_batch = nullptr); + +// This util function is used when opening an existing column family and +// processing its VersionEdit. It does a sanity check for the column family's +// old user comparator and the persist_user_defined_timestamps flag as recorded +// in the VersionEdit, against its new settings from the column family's +// ImmutableCFOptions. +// +// Valid settings change include: +// 1) no user comparator change and no effective persist_user_defined_timestamp +// flag change. +// 2) switch user comparator to enable user-defined timestamps feature provided +// the immediately effective persist_user_defined_timestamps flag is false. +// 3) switch user comparator to disable user-defined timestamps feature provided +// that the before-change persist_user_defined_timestamps is already false. +// +// Switch user comparator to disable/enable UDT is only sanity checked by a user +// comparator name comparison. The full check includes enforcing the new user +// comparator ranks user keys exactly the same as the old user comparator and +// only add / remove the user-defined timestamp comparison. We don't have ways +// to strictly enforce this so currently only the RocksDB builtin comparator +// wrapper `ComparatorWithU64TsImpl` is supported to enable / disable +// user-defined timestamps. It formats user-defined timestamps as uint64_t. +// +// When the settings indicate a legit change to enable user-defined timestamps +// feature on a column family, `mark_sst_files_has_no_udt` will be set to true +// to indicate marking all existing SST files has no user-defined timestamps +// when re-writing the manifest. +Status ValidateUserDefinedTimestampsOptions( + const Comparator* new_comparator, const std::string& old_comparator_name, + bool new_persist_udt, bool old_persist_udt, + bool* mark_sst_files_has_no_udt); + +// Given a cutoff user-defined timestamp formatted as uint64_t, get the +// effective `full_history_ts_low` timestamp, which is the next immediately +// bigger timestamp. Used by the UDT in memtable only feature when flushing +// memtables and remove timestamps. This process collapses history and increase +// the effective `full_history_ts_low`. +void GetFullHistoryTsLowFromU64CutoffTs(Slice* cutoff_ts, + std::string* full_history_ts_low); + +// `start` is the inclusive lower user key bound without user-defined timestamp. +// `end` is the upper user key bound without user-defined timestamp. +// By default, `end` is treated as being exclusive. If `exclusive_end` is set to +// false, it's treated as an inclusive upper bound. +// If any of these two bounds is nullptr, an empty std::optional is +// returned for that bound. +std::tuple, std::optional> +MaybeAddTimestampsToRange(const Slice* start, const Slice* end, size_t ts_sz, + std::string* start_with_ts, std::string* end_with_ts, + bool exclusive_end = true); } // namespace ROCKSDB_NAMESPACE diff --git a/util/udt_util_test.cc b/util/udt_util_test.cc index 63f35992a..44ee567f7 100644 --- a/util/udt_util_test.cc +++ b/util/udt_util_test.cc @@ -321,6 +321,137 @@ TEST_F(HandleTimestampSizeDifferenceTest, UnrecoverableInconsistency) { TimestampSizeConsistencyMode::kReconcileInconsistency) .IsInvalidArgument()); } + +TEST(ValidateUserDefinedTimestampsOptionsTest, EnableUserDefinedTimestamps) { + bool mark_sst_files = false; + const Comparator* new_comparator = test::BytewiseComparatorWithU64TsWrapper(); + const Comparator* old_comparator = BytewiseComparator(); + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + false /*new_persist_udt*/, true /*old_persist_udt*/, &mark_sst_files)); + ASSERT_TRUE(mark_sst_files); + + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + false /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files)); + ASSERT_TRUE(mark_sst_files); +} + +TEST(ValidateUserDefinedTimestampsOptionsTest, + EnableUserDefinedTimestampsNewPersistUDTFlagIncorrect) { + bool mark_sst_files = false; + const Comparator* new_comparator = test::BytewiseComparatorWithU64TsWrapper(); + const Comparator* old_comparator = BytewiseComparator(); + ASSERT_TRUE(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + true /*new_persist_udt*/, true /*old_persist_udt*/, + &mark_sst_files) + .IsInvalidArgument()); + ASSERT_TRUE(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + true /*new_persist_udt*/, false /*old_persist_udt*/, + &mark_sst_files) + .IsInvalidArgument()); +} + +TEST(ValidateUserDefinedTimestampsOptionsTest, DisableUserDefinedTimestamps) { + bool mark_sst_files = false; + const Comparator* new_comparator = ReverseBytewiseComparator(); + const Comparator* old_comparator = + test::ReverseBytewiseComparatorWithU64TsWrapper(); + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + false /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files)); + ASSERT_FALSE(mark_sst_files); + + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + true /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files)); + ASSERT_FALSE(mark_sst_files); +} + +TEST(ValidateUserDefinedTimestampsOptionsTest, + DisableUserDefinedTimestampsOldPersistUDTFlagIncorrect) { + bool mark_sst_files = false; + const Comparator* new_comparator = BytewiseComparator(); + const Comparator* old_comparator = test::BytewiseComparatorWithU64TsWrapper(); + ASSERT_TRUE(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + false /*new_persist_udt*/, true /*old_persist_udt*/, + &mark_sst_files) + .IsInvalidArgument()); + ASSERT_TRUE(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + true /*new_persist_udt*/, true /*old_persist_udt*/, + &mark_sst_files) + .IsInvalidArgument()); +} + +TEST(ValidateUserDefinedTimestampsOptionsTest, UserComparatorUnchanged) { + bool mark_sst_files = false; + const Comparator* ucmp_without_ts = BytewiseComparator(); + const Comparator* ucmp_with_ts = test::BytewiseComparatorWithU64TsWrapper(); + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + ucmp_without_ts, std::string(ucmp_without_ts->Name()), + false /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files)); + ASSERT_FALSE(mark_sst_files); + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + ucmp_without_ts, std::string(ucmp_without_ts->Name()), + true /*new_persist_udt*/, true /*old_persist_udt*/, &mark_sst_files)); + ASSERT_FALSE(mark_sst_files); + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + ucmp_without_ts, std::string(ucmp_without_ts->Name()), + true /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files)); + ASSERT_FALSE(mark_sst_files); + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + ucmp_without_ts, std::string(ucmp_without_ts->Name()), + false /*new_persist_udt*/, true /*old_persist_udt*/, &mark_sst_files)); + ASSERT_FALSE(mark_sst_files); + + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + ucmp_with_ts, std::string(ucmp_with_ts->Name()), true /*new_persist_udt*/, + true /*old_persist_udt*/, &mark_sst_files)); + ASSERT_FALSE(mark_sst_files); + ASSERT_OK(ValidateUserDefinedTimestampsOptions( + ucmp_with_ts, std::string(ucmp_with_ts->Name()), + false /*new_persist_udt*/, false /*old_persist_udt*/, &mark_sst_files)); + ASSERT_FALSE(mark_sst_files); + ASSERT_TRUE(ValidateUserDefinedTimestampsOptions( + ucmp_with_ts, std::string(ucmp_with_ts->Name()), + true /*new_persist_udt*/, false /*old_persist_udt*/, + &mark_sst_files) + .IsInvalidArgument()); + ASSERT_TRUE(ValidateUserDefinedTimestampsOptions( + ucmp_with_ts, std::string(ucmp_with_ts->Name()), + false /*new_persist_udt*/, true /*old_persist_udt*/, + &mark_sst_files) + .IsInvalidArgument()); +} + +TEST(ValidateUserDefinedTimestampsOptionsTest, InvalidUserComparatorChange) { + bool mark_sst_files = false; + const Comparator* new_comparator = BytewiseComparator(); + const Comparator* old_comparator = ReverseBytewiseComparator(); + ASSERT_TRUE(ValidateUserDefinedTimestampsOptions( + new_comparator, std::string(old_comparator->Name()), + false /*new_persist_udt*/, true /*old_persist_udt*/, + &mark_sst_files) + .IsInvalidArgument()); +} + +TEST(GetFullHistoryTsLowFromU64CutoffTsTest, Success) { + std::string cutoff_ts; + uint64_t cutoff_ts_int = 3; + PutFixed64(&cutoff_ts, 3); + Slice cutoff_ts_slice = cutoff_ts; + std::string actual_full_history_ts_low; + GetFullHistoryTsLowFromU64CutoffTs(&cutoff_ts_slice, + &actual_full_history_ts_low); + + std::string expected_ts_low; + PutFixed64(&expected_ts_low, cutoff_ts_int + 1); + ASSERT_EQ(expected_ts_low, actual_full_history_ts_low); +} } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/util/xxhash.h b/util/xxhash.h index ad49bab81..2b9c22883 100644 --- a/util/xxhash.h +++ b/util/xxhash.h @@ -11,9 +11,6 @@ #ifndef XXH_NAMESPACE #define XXH_NAMESPACE ROCKSDB_ #endif // !defined(XXH_NAMESPACE) - -// for FALLTHROUGH_INTENDED, inserted as appropriate -#include "port/lang.h" /* END RocksDB customizations */ // clang-format off diff --git a/util/xxph3.h b/util/xxph3.h index 968000c3a..2933b74db 100644 --- a/util/xxph3.h +++ b/util/xxph3.h @@ -386,10 +386,6 @@ typedef struct { #define XXPH_STATIC_LINKING_ONLY #endif -/* BEGIN RocksDB customizations */ -#include "port/lang.h" /* for FALLTHROUGH_INTENDED, inserted as appropriate */ -/* END RocksDB customizations */ - /* ************************************* * Compiler Specific Options ***************************************/ diff --git a/utilities/backup/backup_engine.cc b/utilities/backup/backup_engine.cc index e74218d45..31a733731 100644 --- a/utilities/backup/backup_engine.cc +++ b/utilities/backup/backup_engine.cc @@ -1583,7 +1583,7 @@ IOStatus BackupEngineImpl::CreateNewBackupWithMetadata( // we copied all the files, enable file deletions if (disabled.ok()) { // If we successfully disabled file deletions - db->EnableFileDeletions(false).PermitUncheckedError(); + db->EnableFileDeletions(/*force=*/false).PermitUncheckedError(); } auto backup_time = backup_env_->NowMicros() - start_backup; diff --git a/utilities/backup/backup_engine_test.cc b/utilities/backup/backup_engine_test.cc index dd41937d4..5ed6ae895 100644 --- a/utilities/backup/backup_engine_test.cc +++ b/utilities/backup/backup_engine_test.cc @@ -589,7 +589,7 @@ void AssertExists(DB* db, int from, int to) { for (int i = from; i < to; ++i) { std::string key = "testkey" + std::to_string(i); std::string value; - Status s = db->Get(ReadOptions(), Slice(key), &value); + ASSERT_OK(db->Get(ReadOptions(), Slice(key), &value)); ASSERT_EQ(value, "testvalue" + std::to_string(i)); } } @@ -4308,13 +4308,13 @@ TEST_F(BackupEngineTest, ExcludeFiles) { for (auto be_pair : {std::make_pair(backup_engine_.get(), alt_backup_engine), std::make_pair(alt_backup_engine, backup_engine_.get())}) { - DestroyDB(dbname_, options_); + ASSERT_OK(DestroyDB(dbname_, options_)); RestoreOptions ro; // Fails without alternate dir ASSERT_TRUE(be_pair.first->RestoreDBFromLatestBackup(dbname_, dbname_, ro) .IsInvalidArgument()); - DestroyDB(dbname_, options_); + ASSERT_OK(DestroyDB(dbname_, options_)); // Works with alternate dir ro.alternate_dirs.push_front(be_pair.second); ASSERT_OK(be_pair.first->RestoreDBFromLatestBackup(dbname_, dbname_, ro)); @@ -4332,7 +4332,7 @@ TEST_F(BackupEngineTest, ExcludeFiles) { for (auto be_pair : {std::make_pair(backup_engine_.get(), alt_backup_engine), std::make_pair(alt_backup_engine, backup_engine_.get())}) { - DestroyDB(dbname_, options_); + ASSERT_OK(DestroyDB(dbname_, options_)); RestoreOptions ro; ro.alternate_dirs.push_front(be_pair.second); ASSERT_OK(be_pair.first->RestoreDBFromLatestBackup(dbname_, dbname_, ro)); diff --git a/utilities/blob_db/blob_db_impl.cc b/utilities/blob_db/blob_db_impl.cc index 7a4b603f2..2fa7ae898 100644 --- a/utilities/blob_db/blob_db_impl.cc +++ b/utilities/blob_db/blob_db_impl.cc @@ -269,7 +269,13 @@ Status BlobDBImpl::Open(std::vector* handles) { // Add trash files in blob dir to file delete scheduler. SstFileManagerImpl* sfm = static_cast( db_impl_->immutable_db_options().sst_file_manager.get()); - DeleteScheduler::CleanupDirectory(env_, sfm, blob_dir_); + s = DeleteScheduler::CleanupDirectory(env_, sfm, blob_dir_); + if (!s.ok()) { + ROCKS_LOG_ERROR(db_options_.info_log, + "Failed to clean up directory %s, status: %s", + blob_dir_.c_str(), s.ToString().c_str()); + return s; + } UpdateLiveSSTSize(); @@ -1142,7 +1148,7 @@ Slice BlobDBImpl::GetCompressedSlice(const Slice& raw, StopWatch compression_sw(clock_, statistics_, BLOB_DB_COMPRESSION_MICROS); CompressionType type = bdb_options_.compression; CompressionOptions opts; - CompressionContext context(type); + CompressionContext context(type, opts); CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), type, 0 /* sample_for_compression */); CompressBlock(raw, info, &type, kBlockBasedTableVersionFormat, false, @@ -1384,28 +1390,46 @@ Status BlobDBImpl::AppendBlob(const std::shared_ptr& bfile, return s; } -std::vector BlobDBImpl::MultiGet(const ReadOptions& read_options, +std::vector BlobDBImpl::MultiGet(const ReadOptions& _read_options, const std::vector& keys, std::vector* values) { StopWatch multiget_sw(clock_, statistics_, BLOB_DB_MULTIGET_MICROS); RecordTick(statistics_, BLOB_DB_NUM_MULTIGET); // Get a snapshot to avoid blob file get deleted between we // fetch and index entry and reading from the file. - ReadOptions ro(read_options); - bool snapshot_created = SetSnapshotIfNeeded(&ro); - std::vector statuses; - statuses.reserve(keys.size()); + std::size_t num_keys = keys.size(); + statuses.reserve(num_keys); + + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + + for (size_t i = 0; i < num_keys; ++i) { + statuses.push_back(s); + } + return statuses; + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } + bool snapshot_created = SetSnapshotIfNeeded(&read_options); + values->clear(); values->reserve(keys.size()); PinnableSlice value; for (size_t i = 0; i < keys.size(); i++) { - statuses.push_back(Get(ro, DefaultColumnFamily(), keys[i], &value)); + statuses.push_back( + GetImpl(read_options, DefaultColumnFamily(), keys[i], &value)); values->push_back(value.ToString()); value.Reset(); } if (snapshot_created) { - db_->ReleaseSnapshot(ro.snapshot); + db_->ReleaseSnapshot(read_options.snapshot); } return statuses; } @@ -1544,12 +1568,12 @@ Status BlobDBImpl::GetRawBlobFromFile(const Slice& key, uint64_t file_number, if (reader->use_direct_io()) { s = reader->Read(IOOptions(), record_offset, static_cast(record_size), &blob_record, nullptr, - &aligned_buf, Env::IO_TOTAL /* rate_limiter_priority */); + &aligned_buf); } else { buf.reserve(static_cast(record_size)); s = reader->Read(IOOptions(), record_offset, static_cast(record_size), &blob_record, &buf[0], - nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + nullptr); } RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, blob_record.size()); } @@ -1609,16 +1633,36 @@ Status BlobDBImpl::GetRawBlobFromFile(const Slice& key, uint64_t file_number, return Status::OK(); } -Status BlobDBImpl::Get(const ReadOptions& read_options, +Status BlobDBImpl::Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) { - return Get(read_options, column_family, key, value, - static_cast(nullptr) /*expiration*/); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { + return Status::InvalidArgument( + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; + } + return GetImpl(read_options, column_family, key, value); } -Status BlobDBImpl::Get(const ReadOptions& read_options, +Status BlobDBImpl::Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value, uint64_t* expiration) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { + return Status::InvalidArgument( + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; + } + StopWatch get_sw(clock_, statistics_, BLOB_DB_GET_MICROS); RecordTick(statistics_, BLOB_DB_NUM_GET); return GetImpl(read_options, column_family, key, value, expiration); @@ -1631,11 +1675,6 @@ Status BlobDBImpl::GetImpl(const ReadOptions& read_options, return Status::NotSupported( "Blob DB doesn't support non-default column family."); } - if (read_options.io_activity != Env::IOActivity::kUnknown) { - return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - } // Get a snapshot to avoid blob file get deleted between we // fetch and index entry and reading from the file. // TODO(yiwu): For Get() retry if file not found would be a simpler strategy. @@ -1882,7 +1921,7 @@ std::pair BlobDBImpl::EvictExpiredFiles(bool aborted) { } if (!blob_file->Immutable()) { - CloseBlobFile(blob_file); + CloseBlobFile(blob_file).PermitUncheckedError(); } assert(blob_file->Immutable()); @@ -2040,11 +2079,16 @@ void BlobDBImpl::CopyBlobFiles( } } -Iterator* BlobDBImpl::NewIterator(const ReadOptions& read_options) { - if (read_options.io_activity != Env::IOActivity::kUnknown) { +Iterator* BlobDBImpl::NewIterator(const ReadOptions& _read_options) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { return NewErrorIterator(Status::InvalidArgument( - "Cannot call NewIterator with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`")); + "Can only call NewIterator with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`")); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; } auto* cfd = static_cast_with_check(DefaultColumnFamily()) @@ -2057,8 +2101,9 @@ Iterator* BlobDBImpl::NewIterator(const ReadOptions& read_options) { own_snapshot = new ManagedSnapshot(db_); snapshot = own_snapshot->snapshot(); } + SuperVersion* sv = cfd->GetReferencedSuperVersion(db_impl_); auto* iter = db_impl_->NewIteratorImpl( - read_options, cfd, snapshot->GetSequenceNumber(), + read_options, cfd, sv, snapshot->GetSequenceNumber(), nullptr /*read_callback*/, true /*expose_blob_index*/); return new BlobDBIterator(own_snapshot, iter, this, clock_, statistics_); } diff --git a/utilities/blob_db/blob_db_impl.h b/utilities/blob_db/blob_db_impl.h index de888c068..d491108d3 100644 --- a/utilities/blob_db/blob_db_impl.h +++ b/utilities/blob_db/blob_db_impl.h @@ -103,12 +103,13 @@ class BlobDBImpl : public BlobDB { const Slice& value) override; using BlobDB::Get; - Status Get(const ReadOptions& read_options, ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* value) override; + Status Get(const ReadOptions& _read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) override; - Status Get(const ReadOptions& read_options, ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* value, - uint64_t* expiration) override; + Status Get(const ReadOptions& _read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, uint64_t* expiration) override; using BlobDB::NewIterator; virtual Iterator* NewIterator(const ReadOptions& read_options) override; @@ -123,7 +124,7 @@ class BlobDBImpl : public BlobDB { using BlobDB::MultiGet; virtual std::vector MultiGet( - const ReadOptions& read_options, const std::vector& keys, + const ReadOptions& _read_options, const std::vector& keys, std::vector* values) override; using BlobDB::Write; @@ -489,7 +490,7 @@ class BlobDBImpl : public BlobDB { // Each call of DisableFileDeletions will increase disable_file_deletion_ // by 1. EnableFileDeletions will either decrease the count by 1 or reset - // it to zeor, depending on the force flag. + // it to zero, depending on the force flag. // // REQUIRES: access with delete_file_mutex_ held. int disable_file_deletions_ = 0; diff --git a/utilities/blob_db/blob_db_listener.h b/utilities/blob_db/blob_db_listener.h index 16aed3340..c95740c50 100644 --- a/utilities/blob_db/blob_db_listener.h +++ b/utilities/blob_db/blob_db_listener.h @@ -22,7 +22,7 @@ class BlobDBListener : public EventListener { void OnFlushBegin(DB* /*db*/, const FlushJobInfo& /*info*/) override { assert(blob_db_impl_ != nullptr); - blob_db_impl_->SyncBlobFiles(); + blob_db_impl_->SyncBlobFiles().PermitUncheckedError(); } void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& /*info*/) override { diff --git a/utilities/blob_db/blob_db_test.cc b/utilities/blob_db/blob_db_test.cc index 015ceb907..07f0cc89e 100644 --- a/utilities/blob_db/blob_db_test.cc +++ b/utilities/blob_db/blob_db_test.cc @@ -2037,7 +2037,7 @@ TEST_F(BlobDBTest, DisableFileDeletions) { ASSERT_EQ(1, blob_db_impl()->TEST_GetObsoleteFiles().size()); VerifyDB(data); // Call EnableFileDeletions a second time. - ASSERT_OK(blob_db_->EnableFileDeletions(false)); + ASSERT_OK(blob_db_->EnableFileDeletions(/*force=*/false)); blob_db_impl()->TEST_DeleteObsoleteFiles(); } // Regardless of value of `force`, file should be deleted by now. diff --git a/utilities/blob_db/blob_dump_tool.cc b/utilities/blob_db/blob_dump_tool.cc index 9b0fa314d..0c2fef5e1 100644 --- a/utilities/blob_db/blob_dump_tool.cc +++ b/utilities/blob_db/blob_dump_tool.cc @@ -102,8 +102,8 @@ Status BlobDumpTool::Read(uint64_t offset, size_t size, Slice* result) { } buffer_.reset(new char[buffer_size_]); } - Status s = reader_->Read(IOOptions(), offset, size, result, buffer_.get(), - nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + Status s = + reader_->Read(IOOptions(), offset, size, result, buffer_.get(), nullptr); if (!s.ok()) { return s; } @@ -277,4 +277,3 @@ std::string BlobDumpTool::GetString(std::pair p) { } // namespace blob_db } // namespace ROCKSDB_NAMESPACE - diff --git a/utilities/blob_db/blob_file.cc b/utilities/blob_db/blob_file.cc index cad89f2e4..5b31d5697 100644 --- a/utilities/blob_db/blob_file.cc +++ b/utilities/blob_db/blob_file.cc @@ -114,13 +114,11 @@ Status BlobFile::ReadFooter(BlobLogFooter* bf) { // TODO: rate limit reading footers from blob files. if (ra_file_reader_->use_direct_io()) { s = ra_file_reader_->Read(IOOptions(), footer_offset, BlobLogFooter::kSize, - &result, nullptr, &aligned_buf, - Env::IO_TOTAL /* rate_limiter_priority */); + &result, nullptr, &aligned_buf); } else { buf.reserve(BlobLogFooter::kSize + 10); s = ra_file_reader_->Read(IOOptions(), footer_offset, BlobLogFooter::kSize, - &result, &buf[0], nullptr, - Env::IO_TOTAL /* rate_limiter_priority */); + &result, &buf[0], nullptr); } if (!s.ok()) return s; if (result.size() != BlobLogFooter::kSize) { @@ -238,13 +236,11 @@ Status BlobFile::ReadMetadata(const std::shared_ptr& fs, // TODO: rate limit reading headers from blob files. if (file_reader->use_direct_io()) { s = file_reader->Read(IOOptions(), 0, BlobLogHeader::kSize, &header_slice, - nullptr, &aligned_buf, - Env::IO_TOTAL /* rate_limiter_priority */); + nullptr, &aligned_buf); } else { header_buf.reserve(BlobLogHeader::kSize); s = file_reader->Read(IOOptions(), 0, BlobLogHeader::kSize, &header_slice, - &header_buf[0], nullptr, - Env::IO_TOTAL /* rate_limiter_priority */); + &header_buf[0], nullptr); } if (!s.ok()) { ROCKS_LOG_ERROR( @@ -281,13 +277,12 @@ Status BlobFile::ReadMetadata(const std::shared_ptr& fs, if (file_reader->use_direct_io()) { s = file_reader->Read(IOOptions(), file_size - BlobLogFooter::kSize, BlobLogFooter::kSize, &footer_slice, nullptr, - &aligned_buf, - Env::IO_TOTAL /* rate_limiter_priority */); + &aligned_buf); } else { footer_buf.reserve(BlobLogFooter::kSize); s = file_reader->Read(IOOptions(), file_size - BlobLogFooter::kSize, BlobLogFooter::kSize, &footer_slice, &footer_buf[0], - nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + nullptr); } if (!s.ok()) { ROCKS_LOG_ERROR( diff --git a/utilities/cache_dump_load_impl.h b/utilities/cache_dump_load_impl.h index 9811ff2d9..59cabbf3b 100644 --- a/utilities/cache_dump_load_impl.h +++ b/utilities/cache_dump_load_impl.h @@ -249,8 +249,7 @@ class FromFileCacheDumpReader : public CacheDumpReader { while (to_read > 0) { io_s = file_reader_->Read(IOOptions(), offset_, to_read, &result_, - buffer_, nullptr, - Env::IO_TOTAL /* rate_limiter_priority */); + buffer_, nullptr); if (!io_s.ok()) { return io_s; } diff --git a/utilities/checkpoint/checkpoint_impl.cc b/utilities/checkpoint/checkpoint_impl.cc index 4a0cc7159..e1f094513 100644 --- a/utilities/checkpoint/checkpoint_impl.cc +++ b/utilities/checkpoint/checkpoint_impl.cc @@ -148,7 +148,7 @@ Status CheckpointImpl::CreateCheckpoint(const std::string& checkpoint_dir, // we copied all the files, enable file deletions if (disabled_file_deletions) { - Status ss = db_->EnableFileDeletions(false); + Status ss = db_->EnableFileDeletions(/*force=*/false); assert(ss.ok()); ss.PermitUncheckedError(); } @@ -337,7 +337,7 @@ Status CheckpointImpl::ExportColumnFamily( nullptr, Temperature::kUnknown); } /*copy_file_cb*/); - const auto enable_status = db_->EnableFileDeletions(false /*force*/); + const auto enable_status = db_->EnableFileDeletions(/*force=*/false); if (s.ok()) { s = enable_status; } diff --git a/utilities/checkpoint/checkpoint_test.cc b/utilities/checkpoint/checkpoint_test.cc index 1cc062667..a9cea1c05 100644 --- a/utilities/checkpoint/checkpoint_test.cc +++ b/utilities/checkpoint/checkpoint_test.cc @@ -925,7 +925,7 @@ TEST_F(CheckpointTest, CheckpointWithDbPath) { options.db_paths.emplace_back(dbname_ + "_2", 0); Reopen(options); ASSERT_OK(Put("key1", "val1")); - Flush(); + ASSERT_OK(Flush()); Checkpoint* checkpoint; ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); // Currently not supported @@ -968,7 +968,7 @@ TEST_F(CheckpointTest, PutRaceWithCheckpointTrackedWalSync) { // Simulate full loss of unsynced data. This drops "key2" -> "val2" from the // DB WAL. - fault_env->DropUnsyncedFileData(); + ASSERT_OK(fault_env->DropUnsyncedFileData()); // Before the bug fix, reopening the DB would fail because the MANIFEST's // AddWal entry indicated the WAL should be synced through "key2" -> "val2". diff --git a/utilities/fault_injection_env.h b/utilities/fault_injection_env.h index 549bfe716..6c1623a8d 100644 --- a/utilities/fault_injection_env.h +++ b/utilities/fault_injection_env.h @@ -96,6 +96,7 @@ class TestWritableFile : public WritableFile { virtual bool use_direct_io() const override { return target_->use_direct_io(); }; + uint64_t GetFileSize() final { return target_->GetFileSize(); } private: FileState state_; diff --git a/utilities/fault_injection_fs.cc b/utilities/fault_injection_fs.cc index fa15fc4a5..53bbaeb07 100644 --- a/utilities/fault_injection_fs.cc +++ b/utilities/fault_injection_fs.cc @@ -408,7 +408,7 @@ IOStatus TestFSRandomAccessFile::Read(uint64_t offset, size_t n, scratch, /*need_count_increase=*/true, /*fault_injected=*/nullptr); } if (s.ok() && fs_->ShouldInjectRandomReadError()) { - return IOStatus::IOError("Injected read error"); + return IOStatus::IOError("injected read error"); } return s; } @@ -430,7 +430,7 @@ IOStatus TestFSRandomAccessFile::ReadAsync( } if (ret.ok()) { if (fs_->ShouldInjectRandomReadError()) { - ret = IOStatus::IOError("Injected read error"); + ret = IOStatus::IOError("injected read error"); } else { s = target_->ReadAsync(req, opts, cb, cb_arg, io_handle, del_fn, nullptr); } @@ -470,7 +470,7 @@ IOStatus TestFSRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs, /*fault_injected=*/nullptr); } if (s.ok() && fs_->ShouldInjectRandomReadError()) { - return IOStatus::IOError("Injected read error"); + return IOStatus::IOError("injected read error"); } return s; } @@ -487,7 +487,7 @@ IOStatus TestFSSequentialFile::Read(size_t n, const IOOptions& options, IODebugContext* dbg) { IOStatus s = target()->Read(n, options, result, scratch, dbg); if (s.ok() && fs_->ShouldInjectRandomReadError()) { - return IOStatus::IOError("Injected seq read error"); + return IOStatus::IOError("injected seq read error"); } return s; } @@ -499,7 +499,7 @@ IOStatus TestFSSequentialFile::PositionedRead(uint64_t offset, size_t n, IOStatus s = target()->PositionedRead(offset, n, options, result, scratch, dbg); if (s.ok() && fs_->ShouldInjectRandomReadError()) { - return IOStatus::IOError("Injected seq positioned read error"); + return IOStatus::IOError("injected seq positioned read error"); } return s; } @@ -678,7 +678,7 @@ IOStatus FaultInjectionTestFS::NewRandomAccessFile( return GetError(); } if (ShouldInjectRandomReadError()) { - return IOStatus::IOError("Injected error when open random access file"); + return IOStatus::IOError("injected error when open random access file"); } IOStatus io_s = InjectThreadSpecificReadError(ErrorOperation::kOpen, nullptr, false, nullptr, @@ -701,7 +701,7 @@ IOStatus FaultInjectionTestFS::NewSequentialFile( } if (ShouldInjectRandomReadError()) { - return IOStatus::IOError("Injected read error when creating seq file"); + return IOStatus::IOError("injected read error when creating seq file"); } IOStatus io_s = target()->NewSequentialFile(fname, file_opts, result, dbg); if (io_s.ok()) { @@ -956,6 +956,7 @@ IOStatus FaultInjectionTestFS::InjectThreadSpecificReadError( return IOStatus::OK(); } + IOStatus ret; if (ctx->rand.OneIn(ctx->one_in)) { if (ctx->count == 0) { ctx->message = ""; @@ -970,15 +971,15 @@ IOStatus FaultInjectionTestFS::InjectThreadSpecificReadError( if (op != ErrorOperation::kMultiReadSingleReq) { // Likely non-per read status code for MultiRead - ctx->message += "error; "; + ctx->message += "injected read error; "; ret_fault_injected = true; - return IOStatus::IOError(); + ret = IOStatus::IOError(ctx->message); } else if (Random::GetTLSInstance()->OneIn(8)) { assert(result); // For a small chance, set the failure to status but turn the // result to be empty, which is supposed to be caught for a check. *result = Slice(); - ctx->message += "inject empty result; "; + ctx->message += "injected empty result; "; ret_fault_injected = true; } else if (!direct_io && Random::GetTLSInstance()->OneIn(7) && scratch != nullptr && result->data() == scratch) { @@ -995,15 +996,18 @@ IOStatus FaultInjectionTestFS::InjectThreadSpecificReadError( // It would work for CRC. Not 100% sure for xxhash and will adjust // if it is not the case. const_cast(result->data())[result->size() - 1]++; - ctx->message += "corrupt last byte; "; + ctx->message += "injected corrupt last byte; "; ret_fault_injected = true; } else { - ctx->message += "error result multiget single; "; + ctx->message += "injected error result multiget single; "; ret_fault_injected = true; - return IOStatus::IOError(); + ret = IOStatus::IOError(ctx->message); } } - return IOStatus::OK(); + if (ctx->retryable) { + ret.SetRetryable(true); + } + return ret; } bool FaultInjectionTestFS::TryParseFileName(const std::string& file_name, @@ -1052,7 +1056,7 @@ IOStatus FaultInjectionTestFS::InjectMetadataWriteError() { } } TEST_SYNC_POINT("FaultInjectionTestFS::InjectMetadataWriteError:Injected"); - return IOStatus::IOError(); + return IOStatus::IOError("injected metadata write error"); } void FaultInjectionTestFS::PrintFaultBacktrace() { diff --git a/utilities/fault_injection_fs.h b/utilities/fault_injection_fs.h index cab0051bd..afd770dde 100644 --- a/utilities/fault_injection_fs.h +++ b/utilities/fault_injection_fs.h @@ -323,8 +323,8 @@ class FaultInjectionTestFS : public FileSystemWrapper { if (!TryParseFileName(file_name, &file_number, &file_type)) { return false; } - return skip_direct_writable_types_.find(file_type) != - skip_direct_writable_types_.end(); + return direct_writable_types_.find(file_type) != + direct_writable_types_.end(); } void SetFilesystemActiveNoLock( bool active, IOStatus error = IOStatus::Corruption("Not active")) { @@ -402,7 +402,8 @@ class FaultInjectionTestFS : public FileSystemWrapper { // seed is the seed for the random number generator, and one_in determines // the probability of injecting error (i.e an error is injected with // 1/one_in probability) - void SetThreadLocalReadErrorContext(uint32_t seed, int one_in) { + void SetThreadLocalReadErrorContext(uint32_t seed, int one_in, + bool retryable) { struct ErrorContext* ctx = static_cast(thread_local_error_->Get()); if (ctx == nullptr) { @@ -411,6 +412,7 @@ class FaultInjectionTestFS : public FileSystemWrapper { } ctx->one_in = one_in; ctx->count = 0; + ctx->retryable = retryable; } static void DeleteThreadLocalErrorContext(void* p) { @@ -437,9 +439,9 @@ class FaultInjectionTestFS : public FileSystemWrapper { write_error_allowed_types_ = types; } - void SetSkipDirectWritableTypes(const std::set& types) { + void SetDirectWritableTypes(const std::set& types) { MutexLock l(&mutex_); - skip_direct_writable_types_ = types; + direct_writable_types_ = types; } void SetRandomMetadataWriteError(int one_in) { @@ -556,12 +558,14 @@ class FaultInjectionTestFS : public FileSystemWrapper { std::string message; int frames; ErrorType type; + bool retryable; explicit ErrorContext(uint32_t seed) : rand(seed), enable_error_injection(false), callstack(nullptr), - frames(0) {} + frames(0), + retryable(false) {} ~ErrorContext() { if (callstack) { free(callstack); @@ -579,7 +583,7 @@ class FaultInjectionTestFS : public FileSystemWrapper { bool inject_for_all_file_types_; std::vector write_error_allowed_types_; // File types where direct writable is skipped. - std::set skip_direct_writable_types_; + std::set direct_writable_types_; bool ingest_data_corruption_before_write_; ChecksumType checksum_handoff_func_tpye_; bool fail_get_file_unique_id_; diff --git a/utilities/fault_injection_secondary_cache.cc b/utilities/fault_injection_secondary_cache.cc index d7a2a1bd7..fa93e8244 100644 --- a/utilities/fault_injection_secondary_cache.cc +++ b/utilities/fault_injection_secondary_cache.cc @@ -78,13 +78,13 @@ FaultInjectionSecondaryCache::GetErrorContext() { Status FaultInjectionSecondaryCache::Insert( const Slice& key, Cache::ObjectPtr value, - const Cache::CacheItemHelper* helper) { + const Cache::CacheItemHelper* helper, bool force_insert) { ErrorContext* ctx = GetErrorContext(); if (ctx->rand.OneIn(prob_)) { return Status::IOError(); } - return base_->Insert(key, value, helper); + return base_->Insert(key, value, helper, force_insert); } std::unique_ptr @@ -92,6 +92,7 @@ FaultInjectionSecondaryCache::Lookup(const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, + Statistics* stats, bool& kept_in_sec_cache) { ErrorContext* ctx = GetErrorContext(); if (base_is_compressed_sec_cache_) { @@ -99,11 +100,12 @@ FaultInjectionSecondaryCache::Lookup(const Slice& key, return nullptr; } else { return base_->Lookup(key, helper, create_context, wait, advise_erase, - kept_in_sec_cache); + stats, kept_in_sec_cache); } } else { - std::unique_ptr hdl = base_->Lookup( - key, helper, create_context, wait, advise_erase, kept_in_sec_cache); + std::unique_ptr hdl = + base_->Lookup(key, helper, create_context, wait, advise_erase, stats, + kept_in_sec_cache); if (wait && ctx->rand.OneIn(prob_)) { hdl.reset(); } diff --git a/utilities/fault_injection_secondary_cache.h b/utilities/fault_injection_secondary_cache.h index ed89f655a..502478d40 100644 --- a/utilities/fault_injection_secondary_cache.h +++ b/utilities/fault_injection_secondary_cache.h @@ -32,12 +32,18 @@ class FaultInjectionSecondaryCache : public SecondaryCache { const char* Name() const override { return "FaultInjectionSecondaryCache"; } Status Insert(const Slice& key, Cache::ObjectPtr value, - const Cache::CacheItemHelper* helper) override; + const Cache::CacheItemHelper* helper, + bool force_insert) override; + + Status InsertSaved(const Slice& /*key*/, const Slice& /*saved*/, + CompressionType /*type*/, CacheTier /*source*/) override { + return Status::OK(); + } std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, - bool& kept_in_sec_cache) override; + Statistics* stats, bool& kept_in_sec_cache) override; bool SupportForceErase() const override { return base_->SupportForceErase(); } diff --git a/utilities/flink/flink_compaction_filter.cc b/utilities/flink/flink_compaction_filter.cc index 4cbdd7e7d..1239e6935 100644 --- a/utilities/flink/flink_compaction_filter.cc +++ b/utilities/flink/flink_compaction_filter.cc @@ -117,13 +117,16 @@ CompactionFilter::Decision FlinkCompactionFilter::FilterV2( const char* data = existing_value.data(); - Debug(logger_.get(), + if (logger_ && logger_->GetInfoLogLevel() <= InfoLogLevel::DEBUG_LEVEL) { + Debug( + logger_.get(), "Call FlinkCompactionFilter::FilterV2 - Key: %s, Data: %s, Value type: " "%d, " "State type: %d, TTL: %" PRId64 " ms, timestamp_offset: %zu", key.ToString().c_str(), existing_value.ToString(true).c_str(), value_type, config_cached_->state_type_, config_cached_->ttl_, config_cached_->timestamp_offset_); + } // too short value to have timestamp at all const bool tooShortValue = diff --git a/utilities/flink/flink_compaction_filter_test.cc b/utilities/flink/flink_compaction_filter_test.cc index 26613ae68..192191cd3 100644 --- a/utilities/flink/flink_compaction_filter_test.cc +++ b/utilities/flink/flink_compaction_filter_test.cc @@ -131,8 +131,8 @@ void Deinit() { delete filter; } TEST(FlinkStateTtlTest, CheckStateTypeEnumOrder) { // NOLINT // if the order changes it also needs to be adjusted in Java client: - // in org.rocksdb.FlinkCompactionFilter - // and in org.rocksdb.FlinkCompactionFilterTest + // in org.forstdb.FlinkCompactionFilter + // and in org.forstdb.FlinkCompactionFilterTest EXPECT_EQ(DISABLED, 0); EXPECT_EQ(VALUE, 1); EXPECT_EQ(LIST, 2); diff --git a/utilities/merge_operators/string_append/stringappend.cc b/utilities/merge_operators/string_append/stringappend.cc index 720dc7200..748e5c89f 100644 --- a/utilities/merge_operators/string_append/stringappend.cc +++ b/utilities/merge_operators/string_append/stringappend.cc @@ -1,8 +1,9 @@ -/** - * A MergeOperator for rocksdb that implements string append. - * @author Deon Nicholas (dnicholas@fb.com) - * Copyright 2013 Facebook - */ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// A MergeOperator for rocksdb that implements string append. #include "stringappend.h" diff --git a/utilities/merge_operators/string_append/stringappend.h b/utilities/merge_operators/string_append/stringappend.h index 153532382..4a7b2b9e5 100644 --- a/utilities/merge_operators/string_append/stringappend.h +++ b/utilities/merge_operators/string_append/stringappend.h @@ -1,8 +1,9 @@ -/** - * A MergeOperator for rocksdb that implements string append. - * @author Deon Nicholas (dnicholas@fb.com) - * Copyright 2013 Facebook - */ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// A MergeOperator for rocksdb that implements string append. #pragma once #include "rocksdb/merge_operator.h" diff --git a/utilities/merge_operators/string_append/stringappend2.cc b/utilities/merge_operators/string_append/stringappend2.cc index 8b6fe9020..bd0716cc3 100644 --- a/utilities/merge_operators/string_append/stringappend2.cc +++ b/utilities/merge_operators/string_append/stringappend2.cc @@ -1,7 +1,7 @@ -/** - * @author Deon Nicholas (dnicholas@fb.com) - * Copyright 2013 Facebook - */ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). #include "stringappend2.h" diff --git a/utilities/merge_operators/string_append/stringappend_test.cc b/utilities/merge_operators/string_append/stringappend_test.cc index 0bf09af8a..acc71c8e4 100644 --- a/utilities/merge_operators/string_append/stringappend_test.cc +++ b/utilities/merge_operators/string_append/stringappend_test.cc @@ -193,6 +193,7 @@ TEST_P(StringAppendOperatorTest, IteratorTest) { ASSERT_EQ(res, "a1,a2,a3"); } } + ASSERT_OK(it->status()); // Should release the snapshot and be aware of the new stuff now it.reset(db_->NewIterator(ReadOptions())); @@ -217,6 +218,7 @@ TEST_P(StringAppendOperatorTest, IteratorTest) { ASSERT_EQ(res, "a1,a2,a3,a4"); } } + ASSERT_OK(it->status()); slists.Append("k3", "g1"); @@ -242,6 +244,7 @@ TEST_P(StringAppendOperatorTest, IteratorTest) { ASSERT_EQ(res, "g1"); } } + ASSERT_OK(it->status()); } TEST_P(StringAppendOperatorTest, SimpleTest) { diff --git a/utilities/option_change_migration/option_change_migration_test.cc b/utilities/option_change_migration/option_change_migration_test.cc index e1f9edc89..1cb42a0ca 100644 --- a/utilities/option_change_migration/option_change_migration_test.cc +++ b/utilities/option_change_migration/option_change_migration_test.cc @@ -87,6 +87,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate1) { for (; it->Valid(); it->Next()) { keys.insert(it->key().ToString()); } + ASSERT_OK(it->status()); } Close(); @@ -124,6 +125,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate1) { it->Next(); } ASSERT_TRUE(!it->Valid()); + ASSERT_OK(it->status()); } } @@ -165,6 +167,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate2) { for (; it->Valid(); it->Next()) { keys.insert(it->key().ToString()); } + ASSERT_OK(it->status()); } Close(); @@ -202,6 +205,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate2) { it->Next(); } ASSERT_TRUE(!it->Valid()); + ASSERT_OK(it->status()); } } @@ -229,7 +233,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate3) { for (int i = 0; i < 50; i++) { ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900))); } - Flush(); + ASSERT_OK(Flush()); ASSERT_OK(dbfull()->TEST_WaitForCompact()); if (num == 9) { // Issue a full compaction to generate some zero-out files @@ -249,6 +253,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate3) { for (; it->Valid(); it->Next()) { keys.insert(it->key().ToString()); } + ASSERT_OK(it->status()); } Close(); @@ -286,6 +291,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate3) { it->Next(); } ASSERT_TRUE(!it->Valid()); + ASSERT_OK(it->status()); } } @@ -313,7 +319,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate4) { for (int i = 0; i < 50; i++) { ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900))); } - Flush(); + ASSERT_OK(Flush()); ASSERT_OK(dbfull()->TEST_WaitForCompact()); if (num == 9) { // Issue a full compaction to generate some zero-out files @@ -333,6 +339,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate4) { for (; it->Valid(); it->Next()) { keys.insert(it->key().ToString()); } + ASSERT_OK(it->status()); } Close(); @@ -370,6 +377,7 @@ TEST_P(DBOptionChangeMigrationTests, Migrate4) { it->Next(); } ASSERT_TRUE(!it->Valid()); + ASSERT_OK(it->status()); } } @@ -496,7 +504,7 @@ TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) { ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900))); } } - Flush(); + ASSERT_OK(Flush()); CompactRangeOptions cro; cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); @@ -509,6 +517,7 @@ TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) { for (; it->Valid(); it->Next()) { keys.insert(it->key().ToString()); } + ASSERT_OK(it->status()); } Close(); diff --git a/utilities/persistent_cache/block_cache_tier_file.cc b/utilities/persistent_cache/block_cache_tier_file.cc index 9a667c1fd..ff01c1abc 100644 --- a/utilities/persistent_cache/block_cache_tier_file.cc +++ b/utilities/persistent_cache/block_cache_tier_file.cc @@ -236,7 +236,7 @@ bool RandomAccessCacheFile::Read(const LBA& lba, Slice* key, Slice* val, Slice result; Status s = freader_->Read(IOOptions(), lba.off_, lba.size_, &result, scratch, - nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + nullptr); if (!s.ok()) { Error(log_, "Error reading from file %s. %s", Path().c_str(), s.ToString().c_str()); @@ -605,4 +605,3 @@ void ThreadedWriter::DispatchIO(const IO& io) { } } // namespace ROCKSDB_NAMESPACE - diff --git a/utilities/simulator_cache/sim_cache.cc b/utilities/simulator_cache/sim_cache.cc index d58c3b34f..ff9d52dca 100644 --- a/utilities/simulator_cache/sim_cache.cc +++ b/utilities/simulator_cache/sim_cache.cc @@ -169,7 +169,8 @@ class SimCacheImpl : public SimCache { Status Insert(const Slice& key, Cache::ObjectPtr value, const CacheItemHelper* helper, size_t charge, Handle** handle, - Priority priority) override { + Priority priority, const Slice& compressed = {}, + CompressionType type = kNoCompression) override { // The handle and value passed in are for real cache, so we pass nullptr // to key_only_cache_ for both instead. Also, the deleter function pointer // will be called by user to perform some external operation which should @@ -178,8 +179,9 @@ class SimCacheImpl : public SimCache { Handle* h = key_only_cache_->Lookup(key); if (h == nullptr) { // TODO: Check for error here? - auto s = key_only_cache_->Insert(key, nullptr, &kNoopCacheItemHelper, - charge, nullptr, priority); + auto s = + key_only_cache_->Insert(key, nullptr, &kNoopCacheItemHelper, charge, + nullptr, priority, compressed, type); s.PermitUncheckedError(); } else { key_only_cache_->Release(h); @@ -189,7 +191,8 @@ class SimCacheImpl : public SimCache { if (!target_) { return Status::OK(); } - return target_->Insert(key, value, helper, charge, handle, priority); + return target_->Insert(key, value, helper, charge, handle, priority, + compressed, type); } Handle* Lookup(const Slice& key, const CacheItemHelper* helper, diff --git a/utilities/trace/file_trace_reader_writer.cc b/utilities/trace/file_trace_reader_writer.cc index 5886d3539..f2ca74144 100644 --- a/utilities/trace/file_trace_reader_writer.cc +++ b/utilities/trace/file_trace_reader_writer.cc @@ -42,8 +42,7 @@ Status FileTraceReader::Reset() { Status FileTraceReader::Read(std::string* data) { assert(file_reader_ != nullptr); Status s = file_reader_->Read(IOOptions(), offset_, kTraceMetadataSize, - &result_, buffer_, nullptr, - Env::IO_TOTAL /* rate_limiter_priority */); + &result_, buffer_, nullptr); if (!s.ok()) { return s; } @@ -68,7 +67,7 @@ Status FileTraceReader::Read(std::string* data) { bytes_to_read > kBufferSize ? kBufferSize : bytes_to_read; while (to_read > 0) { s = file_reader_->Read(IOOptions(), offset_, to_read, &result_, buffer_, - nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + nullptr); if (!s.ok()) { return s; } diff --git a/utilities/transactions/lock/point/point_lock_manager_test.cc b/utilities/transactions/lock/point/point_lock_manager_test.cc index 00cb37d93..28ce5275d 100644 --- a/utilities/transactions/lock/point/point_lock_manager_test.cc +++ b/utilities/transactions/lock/point/point_lock_manager_test.cc @@ -128,14 +128,14 @@ TEST_F(PointLockManagerTest, DeadlockDepthExceeded) { port::Thread t1 = BlockUntilWaitingTxn(wait_sync_point_name_, [&]() { ASSERT_OK(locker_->TryLock(txn2, 1, "k2", env_, true)); // block because txn1 is holding a lock on k1. - locker_->TryLock(txn2, 1, "k1", env_, true); + ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, true)); }); ASSERT_OK(locker_->TryLock(txn3, 1, "k3", env_, true)); port::Thread t2 = BlockUntilWaitingTxn(wait_sync_point_name_, [&]() { // block because txn3 is holding a lock on k1. - locker_->TryLock(txn4, 1, "k3", env_, true); + ASSERT_OK(locker_->TryLock(txn4, 1, "k3", env_, true)); }); auto s = locker_->TryLock(txn3, 1, "k2", env_, true); diff --git a/utilities/transactions/lock/point/point_lock_manager_test.h b/utilities/transactions/lock/point/point_lock_manager_test.h index ca9f46bf9..51d9076b2 100644 --- a/utilities/transactions/lock/point/point_lock_manager_test.h +++ b/utilities/transactions/lock/point/point_lock_manager_test.h @@ -244,7 +244,7 @@ TEST_P(AnyLockManagerTest, Deadlock) { // txn1 tries to lock k2, will block forever. port::Thread t = BlockUntilWaitingTxn(wait_sync_point_name_, [&]() { // block because txn2 is holding a lock on k2. - locker_->TryLock(txn1, 1, "k2", env_, true); + ASSERT_OK(locker_->TryLock(txn1, 1, "k2", env_, true)); }); auto s = locker_->TryLock(txn2, 1, "k1", env_, true); diff --git a/utilities/transactions/lock/range/range_tree/lib/db.h b/utilities/transactions/lock/range/range_tree/lib/db.h index 5aa826c8e..99cfa1f54 100644 --- a/utilities/transactions/lock/range/range_tree/lib/db.h +++ b/utilities/transactions/lock/range/range_tree/lib/db.h @@ -1,3 +1,8 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + #ifndef _DB_H #define _DB_H diff --git a/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h b/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h index af47800fb..c50a3a07f 100644 --- a/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h +++ b/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h @@ -1,3 +1,7 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). // // A replacement for toku_assert.h // diff --git a/utilities/transactions/lock/range/range_tree/lib/portability/toku_external_pthread.h b/utilities/transactions/lock/range/range_tree/lib/portability/toku_external_pthread.h index eb8291c1d..ad1d7bf54 100644 --- a/utilities/transactions/lock/range/range_tree/lib/portability/toku_external_pthread.h +++ b/utilities/transactions/lock/range/range_tree/lib/portability/toku_external_pthread.h @@ -1,3 +1,7 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). /* A wrapper around ROCKSDB_NAMESPACE::TransactionDBMutexFactory-provided condition and mutex that provides toku_pthread_*-like interface. The functions diff --git a/utilities/transactions/lock/range/range_tree/lib/portability/txn_subst.h b/utilities/transactions/lock/range/range_tree/lib/portability/txn_subst.h index 803914862..f4013bb36 100644 --- a/utilities/transactions/lock/range/range_tree/lib/portability/txn_subst.h +++ b/utilities/transactions/lock/range/range_tree/lib/portability/txn_subst.h @@ -1,3 +1,7 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). // // A substitute for ft/txn/txn.h // diff --git a/utilities/transactions/lock/range/range_tree/lib/standalone_port.cc b/utilities/transactions/lock/range/range_tree/lib/standalone_port.cc index 5d0b5228c..6dc86cc99 100644 --- a/utilities/transactions/lock/range/range_tree/lib/standalone_port.cc +++ b/utilities/transactions/lock/range/range_tree/lib/standalone_port.cc @@ -1,3 +1,8 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + #ifndef OS_WIN /* This is a dump ground to make Lock Tree work without the rest of TokuDB. diff --git a/utilities/transactions/lock/range/range_tree/lib/util/partitioned_counter.h b/utilities/transactions/lock/range/range_tree/lib/util/partitioned_counter.h index f20eeedf2..53ca5aab1 100644 --- a/utilities/transactions/lock/range/range_tree/lib/util/partitioned_counter.h +++ b/utilities/transactions/lock/range/range_tree/lib/util/partitioned_counter.h @@ -123,7 +123,7 @@ void partitioned_counters_destroy(void); // Effect: Destroy any partitioned counters data structures. #if defined(__cplusplus) -}; +} #endif #if 0 diff --git a/utilities/transactions/optimistic_transaction_test.cc b/utilities/transactions/optimistic_transaction_test.cc index 46d51956f..733494180 100644 --- a/utilities/transactions/optimistic_transaction_test.cc +++ b/utilities/transactions/optimistic_transaction_test.cc @@ -322,17 +322,11 @@ TEST_P(OptimisticTransactionTest, FlushTest) { delete txn; } -TEST_P(OptimisticTransactionTest, FlushTest2) { - WriteOptions write_options; - ReadOptions read_options, snapshot_read_options; +namespace { +void FlushTest2PopulateTxn(Transaction* txn) { + ReadOptions snapshot_read_options; std::string value; - ASSERT_OK(txn_db->Put(write_options, Slice("foo"), Slice("bar"))); - ASSERT_OK(txn_db->Put(write_options, Slice("foo2"), Slice("bar"))); - - Transaction* txn = txn_db->BeginTransaction(write_options); - ASSERT_NE(txn, nullptr); - snapshot_read_options.snapshot = txn->GetSnapshot(); ASSERT_OK(txn->GetForUpdate(snapshot_read_options, "foo", &value)); @@ -342,6 +336,21 @@ TEST_P(OptimisticTransactionTest, FlushTest2) { ASSERT_OK(txn->GetForUpdate(snapshot_read_options, "foo", &value)); ASSERT_EQ(value, "bar2"); +} +} // namespace + +TEST_P(OptimisticTransactionTest, FlushTest2) { + WriteOptions write_options; + ReadOptions read_options; + std::string value; + + ASSERT_OK(txn_db->Put(write_options, Slice("foo"), Slice("bar"))); + ASSERT_OK(txn_db->Put(write_options, Slice("foo2"), Slice("bar"))); + + Transaction* txn = txn_db->BeginTransaction(write_options); + ASSERT_NE(txn, nullptr); + + FlushTest2PopulateTxn(txn); // Put a random key so we have a MemTable to flush ASSERT_OK(txn_db->Put(write_options, "dummy", "dummy")); @@ -367,9 +376,23 @@ TEST_P(OptimisticTransactionTest, FlushTest2) { // txn should not commit since MemTableList History is not large enough ASSERT_TRUE(s.IsTryAgain()); + // simply trying Commit again doesn't help + s = txn->Commit(); + ASSERT_TRUE(s.IsTryAgain()); + ASSERT_OK(txn_db->Get(read_options, "foo", &value)); ASSERT_EQ(value, "bar"); + // But rolling back and redoing does + ASSERT_OK(txn->Rollback()); + + FlushTest2PopulateTxn(txn); + + ASSERT_OK(txn->Commit()); + + ASSERT_OK(txn_db->Get(read_options, "foo", &value)); + ASSERT_EQ(value, "bar2"); + delete txn; } @@ -681,6 +704,7 @@ TEST_P(OptimisticTransactionTest, ColumnFamiliesTest) { s = txn_db->Get(read_options, "AAA", &value); ASSERT_TRUE(s.IsNotFound()); s = txn_db->Get(read_options, handles[2], "AAAZZZ", &value); + ASSERT_OK(s); ASSERT_EQ(value, "barbar"); Slice key_slices[3] = {Slice("AAA"), Slice("ZZ"), Slice("Z")}; @@ -807,7 +831,7 @@ TEST_P(OptimisticTransactionTest, ColumnFamiliesTest) { cur_seen = {}; txn = txn_db->BeginTransaction(write_options, txn_options); for (const auto& key : keys) { - txn->Put(handles[0], key, "blah"); + ASSERT_OK(txn->Put(handles[0], key, "blah")); } ASSERT_OK(txn->Commit()); // Sufficiently large hash coverage of the space @@ -820,7 +844,7 @@ TEST_P(OptimisticTransactionTest, ColumnFamiliesTest) { cur_seen = {}; txn = txn_db->BeginTransaction(write_options, txn_options, txn); for (const auto& key : keys) { - txn->Put(handles[0], key, "moo"); + ASSERT_OK(txn->Put(handles[0], key, "moo")); } ASSERT_OK(txn->Commit()); ASSERT_EQ(cur_seen.rolling_hash, base_seen.rolling_hash); @@ -831,7 +855,7 @@ TEST_P(OptimisticTransactionTest, ColumnFamiliesTest) { cur_seen = {}; txn = txn_db->BeginTransaction(write_options, txn_options, txn); for (const auto& key : keys) { - txn->Put(handles[1], key, "blah"); + ASSERT_OK(txn->Put(handles[1], key, "blah")); } ASSERT_OK(txn->Commit()); // Different access pattern (different hash seed) @@ -848,7 +872,7 @@ TEST_P(OptimisticTransactionTest, ColumnFamiliesTest) { cur_seen = {}; txn = txn_db->BeginTransaction(write_options, txn_options, txn); for (const auto& key : keys) { - txn->Put(handles[2], key, "blah"); + ASSERT_OK(txn->Put(handles[2], key, "blah")); } ASSERT_OK(txn->Commit()); // Different access pattern (different hash seed) @@ -865,7 +889,7 @@ TEST_P(OptimisticTransactionTest, ColumnFamiliesTest) { delete txn; txn = shared_txn_db->BeginTransaction(write_options, txn_options); for (const auto& key : keys) { - txn->Put(key, "blah"); + ASSERT_OK(txn->Put(key, "blah")); } ASSERT_OK(txn->Commit()); // Different access pattern (different hash seed) @@ -882,7 +906,7 @@ TEST_P(OptimisticTransactionTest, ColumnFamiliesTest) { delete txn; txn = nonshared_txn_db->BeginTransaction(write_options, txn_options); for (const auto& key : keys) { - txn->Put(key, "blah"); + ASSERT_OK(txn->Put(key, "blah")); } ASSERT_OK(txn->Commit()); // Different access pattern (different hash seed) @@ -1399,7 +1423,7 @@ TEST_P(OptimisticTransactionTest, UndoGetForUpdateTest) { txn1->UndoGetForUpdate("A"); Transaction* txn2 = txn_db->BeginTransaction(write_options); - txn2->Put("A", "x"); + ASSERT_OK(txn2->Put("A", "x")); ASSERT_OK(txn2->Commit()); delete txn2; @@ -1611,6 +1635,47 @@ TEST_P(OptimisticTransactionTest, SequenceNumberAfterRecoverTest) { delete transaction; } +#ifdef __SANITIZE_THREAD__ +// Skip OptimisticTransactionTest.SequenceNumberAfterRecoverLargeTest under TSAN +// to avoid false positive because of TSAN lock limit of 64. +#else +TEST_P(OptimisticTransactionTest, SequenceNumberAfterRecoverLargeTest) { + WriteOptions write_options; + OptimisticTransactionOptions transaction_options; + + Transaction* transaction( + txn_db->BeginTransaction(write_options, transaction_options)); + + std::string value(1024 * 1024, 'X'); + const size_t n_zero = 2; + std::string s_i; + Status s; + for (int i = 1; i <= 64; i++) { + s_i = std::to_string(i); + auto key = std::string(n_zero - std::min(n_zero, s_i.length()), '0') + s_i; + s = transaction->Put(key, value); + ASSERT_OK(s); + } + + s = transaction->Commit(); + ASSERT_OK(s); + delete transaction; + + Reopen(); + transaction = txn_db->BeginTransaction(write_options, transaction_options); + s = transaction->Put("bar", "val"); + ASSERT_OK(s); + s = transaction->Commit(); + if (!s.ok()) { + std::cerr << "Failed to commit records. Error: " << s.ToString() + << std::endl; + } + ASSERT_OK(s); + + delete transaction; +} +#endif // __SANITIZE_THREAD__ + TEST_P(OptimisticTransactionTest, TimestampedSnapshotMissingCommitTs) { std::unique_ptr txn(txn_db->BeginTransaction(WriteOptions())); ASSERT_OK(txn->Put("a", "v")); diff --git a/utilities/transactions/pessimistic_transaction.cc b/utilities/transactions/pessimistic_transaction.cc index 1771497a6..1e870190e 100644 --- a/utilities/transactions/pessimistic_transaction.cc +++ b/utilities/transactions/pessimistic_transaction.cc @@ -183,8 +183,8 @@ inline Status WriteCommittedTxn::GetForUpdateImpl( value, exclusive, do_validate); } } else { - Status s = db_impl_->FailIfTsMismatchCf( - column_family, *(read_options.timestamp), /*ts_for_read=*/true); + Status s = + db_impl_->FailIfTsMismatchCf(column_family, *(read_options.timestamp)); if (!s.ok()) { return s; } @@ -888,14 +888,8 @@ Status PessimisticTransaction::LockBatch(WriteBatch* batch, Handler() {} void RecordKey(uint32_t column_family_id, const Slice& key) { - std::string key_str = key.ToString(); - auto& cfh_keys = keys_[column_family_id]; - auto iter = cfh_keys.find(key_str); - if (iter == cfh_keys.end()) { - // key not yet seen, store it. - cfh_keys.insert({std::move(key_str)}); - } + cfh_keys.insert(key.ToString()); } Status PutCF(uint32_t column_family_id, const Slice& key, @@ -1174,4 +1168,16 @@ Status PessimisticTransaction::SetName(const TransactionName& name) { return s; } +Status PessimisticTransaction::CollapseKey(const ReadOptions& options, + const Slice& key, + ColumnFamilyHandle* column_family) { + auto* cfh = column_family ? column_family : db_impl_->DefaultColumnFamily(); + std::string value; + const auto status = GetForUpdate(options, cfh, key, &value, true, true); + if (!status.ok()) { + return status; + } + return Put(column_family, key, value); +} + } // namespace ROCKSDB_NAMESPACE diff --git a/utilities/transactions/pessimistic_transaction.h b/utilities/transactions/pessimistic_transaction.h index ffff50974..bb12266ec 100644 --- a/utilities/transactions/pessimistic_transaction.h +++ b/utilities/transactions/pessimistic_transaction.h @@ -119,6 +119,10 @@ class PessimisticTransaction : public TransactionBaseImpl { const Endpoint& start_key, const Endpoint& end_key) override; + virtual Status CollapseKey( + const ReadOptions& options, const Slice& key, + ColumnFamilyHandle* column_family = nullptr) override; + protected: // Refer to // TransactionOptions::use_only_the_last_commit_time_batch_for_recovery @@ -308,4 +312,3 @@ class WriteCommittedTxn : public PessimisticTransaction { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/utilities/transactions/transaction_base.cc b/utilities/transactions/transaction_base.cc index 5963f7429..b232736cf 100644 --- a/utilities/transactions/transaction_base.cc +++ b/utilities/transactions/transaction_base.cc @@ -110,6 +110,8 @@ void TransactionBaseImpl::Reinitialize(DB* db, start_time_ = dbimpl_->GetSystemClock()->NowMicros(); indexing_enabled_ = true; cmp_ = GetColumnFamilyUserComparator(db_->DefaultColumnFamily()); + WriteBatchInternal::SetDefaultColumnFamilyTimestampSize( + write_batch_.GetWriteBatch(), cmp_->timestamp_size()); WriteBatchInternal::UpdateProtectionInfo( write_batch_.GetWriteBatch(), write_options_.protection_bytes_per_key) .PermitUncheckedError(); @@ -232,27 +234,56 @@ Status TransactionBaseImpl::PopSavePoint() { return write_batch_.PopSavePoint(); } -Status TransactionBaseImpl::Get(const ReadOptions& read_options, +Status TransactionBaseImpl::Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, std::string* value) { - if (read_options.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; + } + auto s = GetImpl(read_options, column_family, key, value); + return s; +} + +Status TransactionBaseImpl::GetImpl(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, + const Slice& key, std::string* value) { assert(value != nullptr); PinnableSlice pinnable_val(value); assert(!pinnable_val.IsPinned()); - auto s = Get(read_options, column_family, key, &pinnable_val); + auto s = GetImpl(read_options, column_family, key, &pinnable_val); if (s.ok() && pinnable_val.IsPinned()) { value->assign(pinnable_val.data(), pinnable_val.size()); } // else value is already assigned return s; } -Status TransactionBaseImpl::Get(const ReadOptions& read_options, +Status TransactionBaseImpl::Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* pinnable_val) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { + return Status::InvalidArgument( + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; + } + return GetImpl(read_options, column_family, key, pinnable_val); +} + +Status TransactionBaseImpl::GetImpl(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, + const Slice& key, + PinnableSlice* pinnable_val) { return write_batch_.GetFromBatchAndDB(db_, read_options, column_family, key, pinnable_val); } @@ -279,7 +310,7 @@ Status TransactionBaseImpl::GetForUpdate(const ReadOptions& read_options, assert(value != nullptr); PinnableSlice pinnable_val(value); assert(!pinnable_val.IsPinned()); - s = Get(read_options, column_family, key, &pinnable_val); + s = GetImpl(read_options, column_family, key, &pinnable_val); if (s.ok() && pinnable_val.IsPinned()) { value->assign(pinnable_val.data(), pinnable_val.size()); } // else value is already assigned @@ -307,39 +338,63 @@ Status TransactionBaseImpl::GetForUpdate(const ReadOptions& read_options, TryLock(column_family, key, true /* read_only */, exclusive, do_validate); if (s.ok() && pinnable_val != nullptr) { - s = Get(read_options, column_family, key, pinnable_val); + s = GetImpl(read_options, column_family, key, pinnable_val); } return s; } std::vector TransactionBaseImpl::MultiGet( - const ReadOptions& read_options, + const ReadOptions& _read_options, const std::vector& column_family, const std::vector& keys, std::vector* values) { size_t num_keys = keys.size(); - if (read_options.io_activity != Env::IOActivity::kUnknown) { + std::vector stat_list(num_keys); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { Status s = Status::InvalidArgument( - "Cannot call MultiGet with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); - return std::vector(num_keys, s); + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + + for (size_t i = 0; i < num_keys; ++i) { + stat_list[i] = s; + } + return stat_list; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; } values->resize(num_keys); - - std::vector stat_list(num_keys); for (size_t i = 0; i < num_keys; ++i) { - stat_list[i] = Get(read_options, column_family[i], keys[i], &(*values)[i]); + stat_list[i] = + GetImpl(read_options, column_family[i], keys[i], &(*values)[i]); } return stat_list; } -void TransactionBaseImpl::MultiGet(const ReadOptions& read_options, +void TransactionBaseImpl::MultiGet(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, const bool sorted_input) { - assert(read_options.io_activity == Env::IOActivity::kUnknown); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } write_batch_.MultiGetFromBatchAndDB(db_, read_options, column_family, num_keys, keys, values, statuses, sorted_input); @@ -349,7 +404,6 @@ std::vector TransactionBaseImpl::MultiGetForUpdate( const ReadOptions& read_options, const std::vector& column_family, const std::vector& keys, std::vector* values) { - // Regardless of whether the MultiGet succeeded, track these keys. size_t num_keys = keys.size(); if (read_options.io_activity != Env::IOActivity::kUnknown) { Status s = Status::InvalidArgument( @@ -357,6 +411,7 @@ std::vector TransactionBaseImpl::MultiGetForUpdate( "`Env::IOActivity::kUnknown`"); return std::vector(num_keys, s); } + // Regardless of whether the MultiGet succeeded, track these keys. values->resize(num_keys); // Lock all keys @@ -372,7 +427,8 @@ std::vector TransactionBaseImpl::MultiGetForUpdate( // TODO(agiardullo): optimize multiget? std::vector stat_list(num_keys); for (size_t i = 0; i < num_keys; ++i) { - stat_list[i] = Get(read_options, column_family[i], keys[i], &(*values)[i]); + stat_list[i] = + GetImpl(read_options, column_family[i], keys[i], &(*values)[i]); } return stat_list; diff --git a/utilities/transactions/transaction_base.h b/utilities/transactions/transaction_base.h index bde09b699..be363b473 100644 --- a/utilities/transactions/transaction_base.h +++ b/utilities/transactions/transaction_base.h @@ -53,11 +53,13 @@ class TransactionBaseImpl : public Transaction { Status PopSavePoint() override; using Transaction::Get; - Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, - const Slice& key, std::string* value) override; + Status Get(const ReadOptions& _read_options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value) override; - Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, - const Slice& key, PinnableSlice* value) override; + Status Get(const ReadOptions& _read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) override; Status Get(const ReadOptions& options, const Slice& key, std::string* value) override { @@ -84,7 +86,7 @@ class TransactionBaseImpl : public Transaction { using Transaction::MultiGet; std::vector MultiGet( - const ReadOptions& options, + const ReadOptions& _read_options, const std::vector& column_family, const std::vector& keys, std::vector* values) override; @@ -98,9 +100,10 @@ class TransactionBaseImpl : public Transaction { keys, values); } - void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family, - const size_t num_keys, const Slice* keys, PinnableSlice* values, - Status* statuses, const bool sorted_input = false) override; + void MultiGet(const ReadOptions& _read_options, + ColumnFamilyHandle* column_family, const size_t num_keys, + const Slice* keys, PinnableSlice* values, Status* statuses, + const bool sorted_input = false) override; using Transaction::MultiGetForUpdate; std::vector MultiGetForUpdate( @@ -260,6 +263,13 @@ class TransactionBaseImpl : public Transaction { LockTracker& GetTrackedLocks() { return *tracked_locks_; } protected: + Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, std::string* value) override; + + virtual Status GetImpl(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) override; + // Add a key to the list of tracked keys. // // seqno is the earliest seqno this key was involved with this transaction. @@ -379,4 +389,3 @@ class TransactionBaseImpl : public Transaction { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/utilities/transactions/transaction_test.cc b/utilities/transactions/transaction_test.cc index ebe924fda..d12626ca8 100644 --- a/utilities/transactions/transaction_test.cc +++ b/utilities/transactions/transaction_test.cc @@ -78,6 +78,112 @@ INSTANTIATE_TEST_CASE_P( std::make_tuple(false, true, WRITE_PREPARED, kUnorderedWrite, true))); #endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +TEST_P(TransactionTest, TestUpperBoundUponDeletion) { + // Reproduction from the original bug report, 11606 + // This test does writes without snapshot validation, and then tries to create + // iterator later, which is unsupported in write unprepared. + if (txn_db_options.write_policy == WRITE_UNPREPARED) { + return; + } + + WriteOptions write_options; + ReadOptions read_options; + Status s; + + Transaction* txn = db->BeginTransaction(write_options); + ASSERT_TRUE(txn); + + // Write some keys in a txn + s = txn->Put("2", "2"); + ASSERT_OK(s); + + s = txn->Put("1", "1"); + ASSERT_OK(s); + + s = txn->Delete("2"); + ASSERT_OK(s); + + read_options.iterate_upper_bound = new Slice("2", 1); + Iterator* iter = txn->GetIterator(read_options); + ASSERT_OK(iter->status()); + iter->SeekToFirst(); + while (iter->Valid()) { + ASSERT_EQ("1", iter->key().ToString()); + iter->Next(); + } + delete iter; + delete txn; + delete read_options.iterate_upper_bound; +} + +TEST_P(TransactionTest, TestTxnRespectBoundsInReadOption) { + if (txn_db_options.write_policy == WRITE_UNPREPARED) { + return; + } + + WriteOptions write_options; + + { + std::unique_ptr txn(db->BeginTransaction(write_options)); + // writes that should be observed by base_iterator_ in BaseDeltaIterator + ASSERT_OK(txn->Put("a", "aa")); + ASSERT_OK(txn->Put("c", "cc")); + ASSERT_OK(txn->Put("e", "ee")); + ASSERT_OK(txn->Put("f", "ff")); + ASSERT_TRUE(txn->Commit().ok()); + } + + std::unique_ptr txn2(db->BeginTransaction(write_options)); + // writes that should be observed by delta_iterator_ in BaseDeltaIterator + ASSERT_OK(txn2->Put("b", "bb")); + ASSERT_OK(txn2->Put("c", "cc")); + ASSERT_OK(txn2->Put("f", "ff")); + + // delta_iterator_: b c f + // base_iterator_: a c e f + // + // given range [c, f) + // assert only {c, e} can be seen + + ReadOptions ro; + ro.iterate_lower_bound = new Slice("c"); + ro.iterate_upper_bound = new Slice("f"); + std::unique_ptr iter(txn2->GetIterator(ro)); + + iter->Seek(Slice("b")); + ASSERT_EQ("c", iter->key()); // lower bound capping + iter->Seek(Slice("f")); + ASSERT_FALSE(iter->Valid()); // out of bound + + iter->SeekForPrev(Slice("f")); + ASSERT_EQ("e", iter->key()); // upper bound capping + iter->SeekForPrev(Slice("b")); + ASSERT_FALSE(iter->Valid()); // out of bound + + // move to the lower bound + iter->SeekToFirst(); + ASSERT_EQ("c", iter->key()); + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + + // move to the upper bound + iter->SeekToLast(); + ASSERT_EQ("e", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + // reversely walk to the beginning + iter->SeekToLast(); + ASSERT_EQ("e", iter->key()); + iter->Prev(); + ASSERT_EQ("c", iter->key()); + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + + delete ro.iterate_lower_bound; + delete ro.iterate_upper_bound; +} + TEST_P(TransactionTest, DoubleEmptyWrite) { WriteOptions write_options; write_options.sync = true; @@ -1382,7 +1488,7 @@ TEST_P(TransactionTest, PersistentTwoPhaseTransactionTest) { ASSERT_OK(db_impl->TEST_FlushMemTable(true)); // regular db read - db->Get(read_options, "foo2", &value); + ASSERT_OK(db->Get(read_options, "foo2", &value)); ASSERT_EQ(value, "bar2"); // nothing has been prepped yet @@ -1430,7 +1536,7 @@ TEST_P(TransactionTest, PersistentTwoPhaseTransactionTest) { ASSERT_OK(s); // value is now available - db->Get(read_options, "foo", &value); + ASSERT_OK(db->Get(read_options, "foo", &value)); ASSERT_EQ(value, "bar"); // we already committed @@ -1601,10 +1707,10 @@ TEST_P(TransactionStressTest, TwoPhaseLongPrepareTest) { // crash fault_fs->SetFilesystemActive(false); reinterpret_cast(db)->TEST_Crash(); - ReOpenNoDelete(); + ASSERT_OK(ReOpenNoDelete()); } else if (i % 37 == 0) { // close - ReOpenNoDelete(); + ASSERT_OK(ReOpenNoDelete()); } } @@ -1668,7 +1774,7 @@ TEST_P(TransactionTest, TwoPhaseSequenceTest) { // kill and reopen fault_fs->SetFilesystemActive(false); - ReOpenNoDelete(); + ASSERT_OK(ReOpenNoDelete()); assert(db != nullptr); // value is now available @@ -1706,7 +1812,7 @@ TEST_P(TransactionTest, TwoPhaseDoubleRecoveryTest) { // kill and reopen fault_fs->SetFilesystemActive(false); reinterpret_cast(db)->TEST_Crash(); - ReOpenNoDelete(); + ASSERT_OK(ReOpenNoDelete()); // commit old txn assert(db != nullptr); // Make clang analyze happy. @@ -2186,9 +2292,9 @@ TEST_P(TransactionTest, WriteConflictTest) { s = txn->Commit(); ASSERT_OK(s); - db->Get(read_options, "foo", &value); + ASSERT_OK(db->Get(read_options, "foo", &value)); ASSERT_EQ(value, "A2"); - db->Get(read_options, "foo2", &value); + ASSERT_OK(db->Get(read_options, "foo2", &value)); ASSERT_EQ(value, "B2"); delete txn; @@ -2230,13 +2336,13 @@ TEST_P(TransactionTest, WriteConflictTest2) { ASSERT_OK(s); // Txn should commit, but only write foo2 and foo3 // Verify that transaction wrote foo2 and foo3 but not foo - db->Get(read_options, "foo", &value); + ASSERT_OK(db->Get(read_options, "foo", &value)); ASSERT_EQ(value, "barz"); - db->Get(read_options, "foo2", &value); + ASSERT_OK(db->Get(read_options, "foo2", &value)); ASSERT_EQ(value, "X"); - db->Get(read_options, "foo3", &value); + ASSERT_OK(db->Get(read_options, "foo3", &value)); ASSERT_EQ(value, "Y"); delete txn; @@ -2328,13 +2434,13 @@ TEST_P(TransactionTest, FlushTest) { // force a memtable flush FlushOptions flush_ops; - db->Flush(flush_ops); + ASSERT_OK(db->Flush(flush_ops)); s = txn->Commit(); // txn should commit since the flushed table is still in MemtableList History ASSERT_OK(s); - db->Get(read_options, "foo", &value); + ASSERT_OK(db->Get(read_options, "foo", &value)); ASSERT_EQ(value, "bar2"); delete txn; @@ -4990,7 +5096,7 @@ TEST_P(TransactionTest, DeleteRangeSupportTest) { } break; case WRITE_PREPARED: - // Intentional fall-through + FALLTHROUGH_INTENDED; case WRITE_UNPREPARED: if (skip_concurrency_control && skip_duplicate_key_check) { ASSERT_OK(s); @@ -6023,7 +6129,7 @@ TEST_P(TransactionTest, DuplicateKeys) { cf_options.max_successive_merges = 2; cf_options.merge_operator = MergeOperators::CreateStringAppendOperator(); ASSERT_OK(ReOpen()); - db->CreateColumnFamily(cf_options, cf_name, &cf_handle); + ASSERT_OK(db->CreateColumnFamily(cf_options, cf_name, &cf_handle)); WriteOptions write_options; // Ensure one value for the key ASSERT_OK(db->Put(write_options, cf_handle, Slice("key"), Slice("value"))); @@ -6759,6 +6865,55 @@ TEST_P(TransactionTest, UnlockWALStallCleared) { } } +TEST_F(TransactionDBTest, CollapseKey) { + ASSERT_OK(ReOpen()); + ASSERT_OK(db->Put({}, "hello", "world")); + ASSERT_OK(db->Flush({})); + ASSERT_OK(db->Merge({}, "hello", "world")); + ASSERT_OK(db->Flush({})); + ASSERT_OK(db->Merge({}, "hello", "world")); + ASSERT_OK(db->Flush({})); + + std::string value; + ASSERT_OK(db->Get({}, "hello", &value)); + ASSERT_EQ("world,world,world", value); + + // get merge op info + std::vector operands(3); + GetMergeOperandsOptions mergeOperandOptions; + mergeOperandOptions.expected_max_number_of_operands = 3; + int numOperands; + ASSERT_OK(db->GetMergeOperands({}, db->DefaultColumnFamily(), "hello", + operands.data(), &mergeOperandOptions, + &numOperands)); + ASSERT_EQ(3, numOperands); + + // collapse key + { + std::unique_ptr txn0{ + db->BeginTransaction(WriteOptions{}, TransactionOptions{})}; + ASSERT_OK(txn0->CollapseKey(ReadOptions{}, "hello")); + ASSERT_OK(txn0->Commit()); + } + + // merge operands should be 1 + ASSERT_OK(db->GetMergeOperands({}, db->DefaultColumnFamily(), "hello", + operands.data(), &mergeOperandOptions, + &numOperands)); + ASSERT_EQ(1, numOperands); + + // get again after collapse + ASSERT_OK(db->Get({}, "hello", &value)); + ASSERT_EQ("world,world,world", value); + + // collapse of non-existent key + { + std::unique_ptr txn1{ + db->BeginTransaction(WriteOptions{}, TransactionOptions{})}; + ASSERT_TRUE(txn1->CollapseKey(ReadOptions{}, "dummy").IsNotFound()); + } +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/utilities/transactions/transaction_test.h b/utilities/transactions/transaction_test.h index e9f53f250..60c5c8a4b 100644 --- a/utilities/transactions/transaction_test.h +++ b/utilities/transactions/transaction_test.h @@ -109,7 +109,7 @@ class TransactionTestBase : public ::testing::Test { delete db; db = nullptr; fault_fs->AssertNoOpenFile(); - fault_fs->DropUnsyncedFileData(); + EXPECT_OK(fault_fs->DropUnsyncedFileData()); fault_fs->ResetState(); Status s; if (use_stackable_db_ == false) { @@ -130,7 +130,7 @@ class TransactionTestBase : public ::testing::Test { delete db; db = nullptr; fault_fs->AssertNoOpenFile(); - fault_fs->DropUnsyncedFileData(); + EXPECT_OK(fault_fs->DropUnsyncedFileData()); fault_fs->ResetState(); Status s; if (use_stackable_db_ == false) { @@ -146,7 +146,7 @@ class TransactionTestBase : public ::testing::Test { Status ReOpen() { delete db; db = nullptr; - DestroyDB(dbname, options); + EXPECT_OK(DestroyDB(dbname, options)); Status s; if (use_stackable_db_ == false) { s = TransactionDB::Open(options, txn_db_options, dbname, &db); @@ -488,6 +488,12 @@ class TransactionTest std::get<2>(GetParam()), std::get<3>(GetParam())){}; }; +class TransactionDBTest : public TransactionTestBase { + public: + TransactionDBTest() + : TransactionTestBase(false, false, WRITE_COMMITTED, kOrderedWrite) {} +}; + class TransactionStressTest : public TransactionTest {}; class MySQLStyleTransactionTest diff --git a/utilities/transactions/write_committed_transaction_ts_test.cc b/utilities/transactions/write_committed_transaction_ts_test.cc index e23851a31..595e7ad1a 100644 --- a/utilities/transactions/write_committed_transaction_ts_test.cc +++ b/utilities/transactions/write_committed_transaction_ts_test.cc @@ -98,6 +98,38 @@ TEST_P(WriteCommittedTxnWithTsTest, SanityChecks) { txn1.reset(); } +void CheckKeyValueTsWithIterator( + Iterator* iter, + std::vector> entries) { + size_t num_entries = entries.size(); + // test forward iteration + for (size_t i = 0; i < num_entries; i++) { + auto [key, value, timestamp] = entries[i]; + if (i == 0) { + iter->Seek(key); + } else { + iter->Next(); + } + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), key); + ASSERT_EQ(iter->value(), value); + ASSERT_EQ(iter->timestamp(), timestamp); + } + // test backward iteration + for (size_t i = 0; i < num_entries; i++) { + auto [key, value, timestamp] = entries[num_entries - 1 - i]; + if (i == 0) { + iter->SeekForPrev(key); + } else { + iter->Prev(); + } + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), key); + ASSERT_EQ(iter->value(), value); + ASSERT_EQ(iter->timestamp(), timestamp); + } +} + TEST_P(WriteCommittedTxnWithTsTest, ReOpenWithTimestamp) { options.merge_operator = MergeOperators::CreateUInt64AddOperator(); ASSERT_OK(ReOpenNoDelete()); @@ -128,17 +160,57 @@ TEST_P(WriteCommittedTxnWithTsTest, ReOpenWithTimestamp) { std::unique_ptr txn1( NewTxn(WriteOptions(), TransactionOptions())); assert(txn1); + + std::string write_ts; + uint64_t write_ts_int = 23; + PutFixed64(&write_ts, write_ts_int); + ReadOptions read_opts; + std::string read_ts; + PutFixed64(&read_ts, write_ts_int + 1); + Slice read_ts_slice = read_ts; + read_opts.timestamp = &read_ts_slice; + + ASSERT_OK(txn1->Put(handles_[1], "bar", "value0")); ASSERT_OK(txn1->Put(handles_[1], "foo", "value1")); + // (key, value, ts) pairs to check. + std::vector> + entries_to_check; + entries_to_check.emplace_back("bar", "value0", ""); + entries_to_check.emplace_back("foo", "value1", ""); + { std::string buf; PutFixed64(&buf, 23); ASSERT_OK(txn1->Put("id", buf)); ASSERT_OK(txn1->Merge("id", buf)); } + + // Check (key, value, ts) with overwrites in txn before `SetCommitTimestamp`. + if (std::get<2>(GetParam())) { // enable_indexing = true + std::unique_ptr iter(txn1->GetIterator(read_opts, handles_[1])); + CheckKeyValueTsWithIterator(iter.get(), entries_to_check); + } + ASSERT_OK(txn1->SetName("txn1")); ASSERT_OK(txn1->Prepare()); - ASSERT_OK(txn1->SetCommitTimestamp(/*ts=*/23)); + ASSERT_OK(txn1->SetCommitTimestamp(write_ts_int)); + + // Check (key, value, ts) with overwrites in txn after `SetCommitTimestamp`. + if (std::get<2>(GetParam())) { // enable_indexing = true + std::unique_ptr iter(txn1->GetIterator(read_opts, handles_[1])); + CheckKeyValueTsWithIterator(iter.get(), entries_to_check); + } + ASSERT_OK(txn1->Commit()); + entries_to_check.clear(); + entries_to_check.emplace_back("bar", "value0", write_ts); + entries_to_check.emplace_back("foo", "value1", write_ts); + + // Check (key, value, ts) pairs with overwrites in txn after `Commit`. + { + std::unique_ptr iter(txn1->GetIterator(read_opts, handles_[1])); + CheckKeyValueTsWithIterator(iter.get(), entries_to_check); + } txn1.reset(); { @@ -159,6 +231,14 @@ TEST_P(WriteCommittedTxnWithTsTest, ReOpenWithTimestamp) { assert(result); ASSERT_EQ(46, ival); } + + // Check (key, value, ts) pairs without overwrites in txn. + { + std::unique_ptr txn2( + NewTxn(WriteOptions(), TransactionOptions())); + std::unique_ptr iter(txn2->GetIterator(read_opts, handles_[1])); + CheckKeyValueTsWithIterator(iter.get(), entries_to_check); + } } TEST_P(WriteCommittedTxnWithTsTest, RecoverFromWal) { @@ -564,6 +644,13 @@ TEST_P(WriteCommittedTxnWithTsTest, CheckKeysForConflicts) { ASSERT_TRUE(txn1->GetForUpdate(ReadOptions(), "foo", &dontcare).IsBusy()); ASSERT_TRUE(called); + Transaction* reused_txn = + db->BeginTransaction(WriteOptions(), TransactionOptions(), txn1.get()); + ASSERT_EQ(reused_txn, txn1.get()); + ASSERT_OK(reused_txn->Put("foo", "v1")); + ASSERT_OK(reused_txn->SetCommitTimestamp(40)); + ASSERT_OK(reused_txn->Commit()); + SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); } diff --git a/utilities/transactions/write_prepared_transaction_test.cc b/utilities/transactions/write_prepared_transaction_test.cc index 4041b6ad8..8427e2d7c 100644 --- a/utilities/transactions/write_prepared_transaction_test.cc +++ b/utilities/transactions/write_prepared_transaction_test.cc @@ -1345,7 +1345,7 @@ TEST_P(WritePreparedTransactionTest, NewSnapshotLargerThanMax) { // Check that the new max has not advanced the last seq ASSERT_LT(wp_db->max_evicted_seq_.load(), last_seq); for (auto txn : txns) { - txn->Rollback(); + ASSERT_OK(txn->Rollback()); delete txn; } } diff --git a/utilities/transactions/write_prepared_txn.cc b/utilities/transactions/write_prepared_txn.cc index c27a679e4..58126a475 100644 --- a/utilities/transactions/write_prepared_txn.cc +++ b/utilities/transactions/write_prepared_txn.cc @@ -39,20 +39,37 @@ void WritePreparedTxn::Initialize(const TransactionOptions& txn_options) { prepare_batch_cnt_ = 0; } -void WritePreparedTxn::MultiGet(const ReadOptions& options, +void WritePreparedTxn::MultiGet(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, const bool sorted_input) { - assert(options.io_activity == Env::IOActivity::kUnknown); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } + SequenceNumber min_uncommitted, snap_seq; - const SnapshotBackup backed_by_snapshot = - wpt_db_->AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq); + const SnapshotBackup backed_by_snapshot = wpt_db_->AssignMinMaxSeqs( + read_options.snapshot, &min_uncommitted, &snap_seq); WritePreparedTxnReadCallback callback(wpt_db_, snap_seq, min_uncommitted, backed_by_snapshot); - write_batch_.MultiGetFromBatchAndDB(db_, options, column_family, num_keys, - keys, values, statuses, sorted_input, - &callback); + write_batch_.MultiGetFromBatchAndDB(db_, read_options, column_family, + num_keys, keys, values, statuses, + sorted_input, &callback); if (UNLIKELY(!callback.valid() || !wpt_db_->ValidateSnapshot(snap_seq, backed_by_snapshot))) { wpt_db_->WPRecordTick(TXN_GET_TRY_AGAIN); @@ -62,14 +79,27 @@ void WritePreparedTxn::MultiGet(const ReadOptions& options, } } -Status WritePreparedTxn::Get(const ReadOptions& options, +Status WritePreparedTxn::Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* pinnable_val) { - if (options.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; + } + + return GetImpl(read_options, column_family, key, pinnable_val); +} + +Status WritePreparedTxn::GetImpl(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key, + PinnableSlice* pinnable_val) { SequenceNumber min_uncommitted, snap_seq; const SnapshotBackup backed_by_snapshot = wpt_db_->AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq); @@ -93,11 +123,7 @@ Status WritePreparedTxn::Get(const ReadOptions& options, } Iterator* WritePreparedTxn::GetIterator(const ReadOptions& options) { - // Make sure to get iterator from WritePrepareTxnDB, not the root db. - Iterator* db_iter = wpt_db_->NewIterator(options); - assert(db_iter); - - return write_batch_.NewIteratorWithBase(db_iter); + return GetIterator(options, wpt_db_->DefaultColumnFamily()); } Iterator* WritePreparedTxn::GetIterator(const ReadOptions& options, @@ -106,7 +132,7 @@ Iterator* WritePreparedTxn::GetIterator(const ReadOptions& options, Iterator* db_iter = wpt_db_->NewIterator(options, column_family); assert(db_iter); - return write_batch_.NewIteratorWithBase(column_family, db_iter); + return write_batch_.NewIteratorWithBase(column_family, db_iter, &options); } Status WritePreparedTxn::PrepareInternal() { diff --git a/utilities/transactions/write_prepared_txn.h b/utilities/transactions/write_prepared_txn.h index f621e37ab..9a0fb81d1 100644 --- a/utilities/transactions/write_prepared_txn.h +++ b/utilities/transactions/write_prepared_txn.h @@ -51,12 +51,12 @@ class WritePreparedTxn : public PessimisticTransaction { // seq in the WAL that is also published, LastPublishedSequence, as opposed to // the last seq in the memtable. using Transaction::Get; - virtual Status Get(const ReadOptions& options, + virtual Status Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) override; using Transaction::MultiGet; - virtual void MultiGet(const ReadOptions& options, + virtual void MultiGet(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, @@ -86,6 +86,10 @@ class WritePreparedTxn : public PessimisticTransaction { friend class WriteUnpreparedTxnDB; friend class WriteUnpreparedTxn; + using Transaction::GetImpl; + Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value) override; + Status PrepareInternal() override; Status CommitWithoutPrepareInternal() override; @@ -114,4 +118,3 @@ class WritePreparedTxn : public PessimisticTransaction { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/utilities/transactions/write_prepared_txn_db.cc b/utilities/transactions/write_prepared_txn_db.cc index 6118c3549..91a81d158 100644 --- a/utilities/transactions/write_prepared_txn_db.cc +++ b/utilities/transactions/write_prepared_txn_db.cc @@ -247,14 +247,26 @@ Status WritePreparedTxnDB::WriteInternal(const WriteOptions& write_options_orig, return s; } -Status WritePreparedTxnDB::Get(const ReadOptions& options, +Status WritePreparedTxnDB::Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) { - if (options.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; + } + + return GetImpl(read_options, column_family, key, value); +} + +Status WritePreparedTxnDB::GetImpl(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value) { SequenceNumber min_uncommitted, snap_seq; const SnapshotBackup backed_by_snapshot = AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq); @@ -314,16 +326,35 @@ void WritePreparedTxnDB::UpdateCFComparatorMap(ColumnFamilyHandle* h) { } std::vector WritePreparedTxnDB::MultiGet( - const ReadOptions& options, + const ReadOptions& _read_options, const std::vector& column_family, const std::vector& keys, std::vector* values) { assert(values); size_t num_keys = keys.size(); + std::vector stat_list(num_keys); + + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + + for (size_t i = 0; i < num_keys; ++i) { + stat_list[i] = s; + } + return stat_list; + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } + values->resize(num_keys); - std::vector stat_list(num_keys); for (size_t i = 0; i < num_keys; ++i) { - stat_list[i] = this->Get(options, column_family[i], keys[i], &(*values)[i]); + stat_list[i] = + this->GetImpl(read_options, column_family[i], keys[i], &(*values)[i]); } return stat_list; } @@ -346,22 +377,27 @@ static void CleanupWritePreparedTxnDBIterator(void* arg1, void* /*arg2*/) { } } // anonymous namespace -Iterator* WritePreparedTxnDB::NewIterator(const ReadOptions& options, +Iterator* WritePreparedTxnDB::NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) { - if (options.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { return NewErrorIterator(Status::InvalidArgument( - "Cannot call NewIterator with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`")); + "Can only call NewIterator with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`")); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; } constexpr bool expose_blob_index = false; constexpr bool allow_refresh = false; std::shared_ptr own_snapshot = nullptr; SequenceNumber snapshot_seq = kMaxSequenceNumber; SequenceNumber min_uncommitted = 0; - if (options.snapshot != nullptr) { - snapshot_seq = options.snapshot->GetSequenceNumber(); + if (read_options.snapshot != nullptr) { + snapshot_seq = read_options.snapshot->GetSequenceNumber(); min_uncommitted = - static_cast_with_check(options.snapshot) + static_cast_with_check(read_options.snapshot) ->min_uncommitted_; } else { auto* snapshot = GetSnapshot(); @@ -377,26 +413,38 @@ Iterator* WritePreparedTxnDB::NewIterator(const ReadOptions& options, static_cast_with_check(column_family)->cfd(); auto* state = new IteratorState(this, snapshot_seq, own_snapshot, min_uncommitted); - auto* db_iter = - db_impl_->NewIteratorImpl(options, cfd, snapshot_seq, &state->callback, - expose_blob_index, allow_refresh); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl_); + auto* db_iter = db_impl_->NewIteratorImpl(read_options, cfd, super_version, + snapshot_seq, &state->callback, + expose_blob_index, allow_refresh); db_iter->RegisterCleanup(CleanupWritePreparedTxnDBIterator, state, nullptr); return db_iter; } Status WritePreparedTxnDB::NewIterators( - const ReadOptions& options, + const ReadOptions& _read_options, const std::vector& column_families, std::vector* iterators) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { + return Status::InvalidArgument( + "Can only call NewIterator with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`"); + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; + } constexpr bool expose_blob_index = false; constexpr bool allow_refresh = false; std::shared_ptr own_snapshot = nullptr; SequenceNumber snapshot_seq = kMaxSequenceNumber; SequenceNumber min_uncommitted = 0; - if (options.snapshot != nullptr) { - snapshot_seq = options.snapshot->GetSequenceNumber(); + if (read_options.snapshot != nullptr) { + snapshot_seq = read_options.snapshot->GetSequenceNumber(); min_uncommitted = - static_cast_with_check(options.snapshot) + static_cast_with_check(read_options.snapshot) ->min_uncommitted_; } else { auto* snapshot = GetSnapshot(); @@ -414,9 +462,10 @@ Status WritePreparedTxnDB::NewIterators( static_cast_with_check(column_family)->cfd(); auto* state = new IteratorState(this, snapshot_seq, own_snapshot, min_uncommitted); - auto* db_iter = - db_impl_->NewIteratorImpl(options, cfd, snapshot_seq, &state->callback, - expose_blob_index, allow_refresh); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl_); + auto* db_iter = db_impl_->NewIteratorImpl(read_options, cfd, super_version, + snapshot_seq, &state->callback, + expose_blob_index, allow_refresh); db_iter->RegisterCleanup(CleanupWritePreparedTxnDBIterator, state, nullptr); iterators->push_back(db_iter); } diff --git a/utilities/transactions/write_prepared_txn_db.h b/utilities/transactions/write_prepared_txn_db.h index f5b641160..6c0e292b0 100644 --- a/utilities/transactions/write_prepared_txn_db.h +++ b/utilities/transactions/write_prepared_txn_db.h @@ -83,24 +83,24 @@ class WritePreparedTxnDB : public PessimisticTransactionDB { size_t batch_cnt, WritePreparedTxn* txn); using DB::Get; - virtual Status Get(const ReadOptions& options, + virtual Status Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) override; using DB::MultiGet; virtual std::vector MultiGet( - const ReadOptions& options, + const ReadOptions& _read_options, const std::vector& column_family, const std::vector& keys, std::vector* values) override; using DB::NewIterator; - virtual Iterator* NewIterator(const ReadOptions& options, + virtual Iterator* NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) override; using DB::NewIterators; virtual Status NewIterators( - const ReadOptions& options, + const ReadOptions& _read_options, const std::vector& column_families, std::vector* iterators) override; @@ -520,6 +520,21 @@ class WritePreparedTxnDB : public PessimisticTransactionDB { RecordTick(db_impl_->immutable_db_options_.statistics.get(), ticker_type); } + Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, std::string* value) { + assert(value != nullptr); + PinnableSlice pinnable_val(value); + assert(!pinnable_val.IsPinned()); + auto s = GetImpl(options, column_family, key, &pinnable_val); + if (s.ok() && pinnable_val.IsPinned()) { + value->assign(pinnable_val.data(), pinnable_val.size()); + } // else value is already assigned + return s; + } + + Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value); + // A heap with the amortized O(1) complexity for erase. It uses one extra heap // to keep track of erased entries that are not yet on top of the main heap. class PreparedHeap { diff --git a/utilities/transactions/write_unprepared_txn.cc b/utilities/transactions/write_unprepared_txn.cc index 845b117cf..c30cf9e1f 100644 --- a/utilities/transactions/write_unprepared_txn.cc +++ b/utilities/transactions/write_unprepared_txn.cc @@ -943,20 +943,36 @@ Status WriteUnpreparedTxn::PopSavePoint() { return Status::NotFound(); } -void WriteUnpreparedTxn::MultiGet(const ReadOptions& options, +void WriteUnpreparedTxn::MultiGet(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, const bool sorted_input) { - assert(options.io_activity == Env::IOActivity::kUnknown); + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kMultiGet) { + Status s = Status::InvalidArgument( + "Can only call MultiGet with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`"); + + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = s; + } + } + return; + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kMultiGet; + } SequenceNumber min_uncommitted, snap_seq; - const SnapshotBackup backed_by_snapshot = - wupt_db_->AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq); + const SnapshotBackup backed_by_snapshot = wupt_db_->AssignMinMaxSeqs( + read_options.snapshot, &min_uncommitted, &snap_seq); WriteUnpreparedTxnReadCallback callback(wupt_db_, snap_seq, min_uncommitted, unprep_seqs_, backed_by_snapshot); - write_batch_.MultiGetFromBatchAndDB(db_, options, column_family, num_keys, - keys, values, statuses, sorted_input, - &callback); + write_batch_.MultiGetFromBatchAndDB(db_, read_options, column_family, + num_keys, keys, values, statuses, + sorted_input, &callback); if (UNLIKELY(!callback.valid() || !wupt_db_->ValidateSnapshot(snap_seq, backed_by_snapshot))) { wupt_db_->WPRecordTick(TXN_GET_TRY_AGAIN); @@ -966,14 +982,26 @@ void WriteUnpreparedTxn::MultiGet(const ReadOptions& options, } } -Status WriteUnpreparedTxn::Get(const ReadOptions& options, +Status WriteUnpreparedTxn::Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) { - if (options.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kGet) { return Status::InvalidArgument( - "Cannot call Get with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`"); + "Can only call Get with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`"); + } + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kGet; } + + return GetImpl(read_options, column_family, key, value); +} + +Status WriteUnpreparedTxn::GetImpl(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value) { SequenceNumber min_uncommitted, snap_seq; const SnapshotBackup backed_by_snapshot = wupt_db_->AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq); @@ -1009,7 +1037,8 @@ Iterator* WriteUnpreparedTxn::GetIterator(const ReadOptions& options, Iterator* db_iter = wupt_db_->NewIterator(options, column_family, this); assert(db_iter); - auto iter = write_batch_.NewIteratorWithBase(column_family, db_iter); + auto iter = + write_batch_.NewIteratorWithBase(column_family, db_iter, &options); active_iterators_.push_back(iter); iter->RegisterCleanup(CleanupWriteUnpreparedWBWIIterator, this, iter); return iter; diff --git a/utilities/transactions/write_unprepared_txn.h b/utilities/transactions/write_unprepared_txn.h index 63c65f00a..fe47c8cd8 100644 --- a/utilities/transactions/write_unprepared_txn.h +++ b/utilities/transactions/write_unprepared_txn.h @@ -184,12 +184,12 @@ class WriteUnpreparedTxn : public WritePreparedTxn { // Get and GetIterator needs to be overridden so that a ReadCallback to // handle read-your-own-write is used. using Transaction::Get; - virtual Status Get(const ReadOptions& options, + virtual Status Get(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) override; using Transaction::MultiGet; - virtual void MultiGet(const ReadOptions& options, + virtual void MultiGet(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, @@ -211,6 +211,10 @@ class WriteUnpreparedTxn : public WritePreparedTxn { friend class WriteUnpreparedTxnDB; const std::map& GetUnpreparedSequenceNumbers(); + using Transaction::GetImpl; + Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value) override; + Status WriteRollbackKeys(const LockTracker& tracked_keys, WriteBatchWithIndex* rollback_batch, ReadCallback* callback, const ReadOptions& roptions); @@ -336,4 +340,3 @@ class WriteUnpreparedTxn : public WritePreparedTxn { }; } // namespace ROCKSDB_NAMESPACE - diff --git a/utilities/transactions/write_unprepared_txn_db.cc b/utilities/transactions/write_unprepared_txn_db.cc index fd0ba0aed..1d75dd449 100644 --- a/utilities/transactions/write_unprepared_txn_db.cc +++ b/utilities/transactions/write_unprepared_txn_db.cc @@ -385,13 +385,19 @@ static void CleanupWriteUnpreparedTxnDBIterator(void* arg1, void* /*arg2*/) { } } // anonymous namespace -Iterator* WriteUnpreparedTxnDB::NewIterator(const ReadOptions& options, +Iterator* WriteUnpreparedTxnDB::NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, WriteUnpreparedTxn* txn) { - if (options.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { return NewErrorIterator(Status::InvalidArgument( - "Cannot call NewIterator with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`")); + "Can only call NewIterator with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`")); + } + + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; } // TODO(lth): Refactor so that this logic is shared with WritePrepared. constexpr bool expose_blob_index = false; @@ -431,11 +437,11 @@ Iterator* WriteUnpreparedTxnDB::NewIterator(const ReadOptions& options, // max_visible_seq, and then return the last visible value, so that this // restriction can be lifted. const Snapshot* snapshot = nullptr; - if (options.snapshot == nullptr) { + if (read_options.snapshot == nullptr) { snapshot = GetSnapshot(); own_snapshot = std::make_shared(db_impl_, snapshot); } else { - snapshot = options.snapshot; + snapshot = read_options.snapshot; } snapshot_seq = snapshot->GetSequenceNumber(); @@ -466,9 +472,10 @@ Iterator* WriteUnpreparedTxnDB::NewIterator(const ReadOptions& options, static_cast_with_check(column_family)->cfd(); auto* state = new IteratorState(this, snapshot_seq, own_snapshot, min_uncommitted, txn); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl_); auto* db_iter = db_impl_->NewIteratorImpl( - options, cfd, state->MaxVisibleSeq(), &state->callback, expose_blob_index, - allow_refresh); + read_options, cfd, super_version, state->MaxVisibleSeq(), + &state->callback, expose_blob_index, allow_refresh); db_iter->RegisterCleanup(CleanupWriteUnpreparedTxnDBIterator, state, nullptr); return db_iter; } diff --git a/utilities/transactions/write_unprepared_txn_db.h b/utilities/transactions/write_unprepared_txn_db.h index a7b10f153..409d73a0a 100644 --- a/utilities/transactions/write_unprepared_txn_db.h +++ b/utilities/transactions/write_unprepared_txn_db.h @@ -27,7 +27,7 @@ class WriteUnpreparedTxnDB : public WritePreparedTxnDB { struct IteratorState; using WritePreparedTxnDB::NewIterator; - Iterator* NewIterator(const ReadOptions& options, + Iterator* NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, WriteUnpreparedTxn* txn); diff --git a/utilities/ttl/db_ttl_impl.cc b/utilities/ttl/db_ttl_impl.cc index 2b261ec6f..e4bff7826 100644 --- a/utilities/ttl/db_ttl_impl.cc +++ b/utilities/ttl/db_ttl_impl.cc @@ -19,9 +19,9 @@ namespace ROCKSDB_NAMESPACE { static std::unordered_map ttl_merge_op_type_info = - {{"user_operator", - OptionTypeInfo::AsCustomSharedPtr( - 0, OptionVerificationType::kByName, OptionTypeFlags::kNone)}}; + {{"user_operator", OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByNameAllowNull, + OptionTypeFlags::kNone)}}; TtlMergeOperator::TtlMergeOperator( const std::shared_ptr& merge_op, SystemClock* clock) @@ -594,14 +594,19 @@ Status DBWithTTLImpl::Write(const WriteOptions& opts, WriteBatch* updates) { } } -Iterator* DBWithTTLImpl::NewIterator(const ReadOptions& opts, +Iterator* DBWithTTLImpl::NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) { - if (opts.io_activity != Env::IOActivity::kUnknown) { + if (_read_options.io_activity != Env::IOActivity::kUnknown && + _read_options.io_activity != Env::IOActivity::kDBIterator) { return NewErrorIterator(Status::InvalidArgument( - "Cannot call NewIterator with `ReadOptions::io_activity` != " - "`Env::IOActivity::kUnknown`")); + "Can only call NewIterator with `ReadOptions::io_activity` is " + "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`")); } - return new TtlIterator(db_->NewIterator(opts, column_family)); + ReadOptions read_options(_read_options); + if (read_options.io_activity == Env::IOActivity::kUnknown) { + read_options.io_activity = Env::IOActivity::kDBIterator; + } + return new TtlIterator(db_->NewIterator(read_options, column_family)); } void DBWithTTLImpl::SetTtl(ColumnFamilyHandle* h, int32_t ttl) { diff --git a/utilities/ttl/db_ttl_impl.h b/utilities/ttl/db_ttl_impl.h index 6ac662467..b125d79b0 100644 --- a/utilities/ttl/db_ttl_impl.h +++ b/utilities/ttl/db_ttl_impl.h @@ -78,7 +78,7 @@ class DBWithTTLImpl : public DBWithTTL { virtual Status Write(const WriteOptions& opts, WriteBatch* updates) override; using StackableDB::NewIterator; - virtual Iterator* NewIterator(const ReadOptions& opts, + virtual Iterator* NewIterator(const ReadOptions& _read_options, ColumnFamilyHandle* column_family) override; virtual DB* GetBaseDB() override { return db_; } diff --git a/utilities/ttl/ttl_test.cc b/utilities/ttl/ttl_test.cc index 225db59b5..da1d2d0da 100644 --- a/utilities/ttl/ttl_test.cc +++ b/utilities/ttl/ttl_test.cc @@ -403,8 +403,10 @@ class TtlTest : public testing::Test { DBWithTTL* db_ttl_; std::unique_ptr env_; - private: + protected: Options options_; + + private: KVMap kvmap_; KVMap::iterator kv_it_; const std::string kNewValue_ = "new_value"; @@ -611,6 +613,17 @@ TEST_F(TtlTest, CompactionFilter) { CloseTtl(); } +TEST_F(TtlTest, UnregisteredMergeOperator) { + class UnregisteredMergeOperator : public MergeOperator { + public: + const char* Name() const override { return "UnregisteredMergeOperator"; } + }; + options_.fail_if_options_file_error = true; + options_.merge_operator = std::make_shared(); + OpenTtl(); + CloseTtl(); +} + // Insert some key-values which KeyMayExist should be able to get and check that // values returned are fine TEST_F(TtlTest, KeyMayExist) { @@ -652,7 +665,7 @@ TEST_F(TtlTest, ColumnFamiliesTest) { options.create_if_missing = true; options.env = env_.get(); - DB::Open(options, dbname_, &db); + ASSERT_OK(DB::Open(options, dbname_, &db)); ColumnFamilyHandle* handle; ASSERT_OK(db->CreateColumnFamily(ColumnFamilyOptions(options), "ttl_column_family", &handle)); @@ -901,6 +914,14 @@ TEST_F(TtlOptionsTest, LoadTtlMergeOperator) { std::shared_ptr copy; ASSERT_OK(MergeOperator::CreateFromString(config_options_, opts_str, ©)); ASSERT_TRUE(mo->AreEquivalent(config_options_, copy.get(), &mismatch)); + + // An unregistered user_operator will be null, which is not supported by the + // `TtlMergeOperator` implementation. + ASSERT_OK(MergeOperator::CreateFromString( + config_options_, "id=TtlMergeOperator; user_operator=unknown", &mo)); + ASSERT_NE(mo.get(), nullptr); + ASSERT_STREQ(mo->Name(), TtlMergeOperator::kClassName()); + ASSERT_NOK(mo->ValidateOptions(DBOptions(), ColumnFamilyOptions())); } } // namespace ROCKSDB_NAMESPACE diff --git a/utilities/write_batch_with_index/write_batch_with_index.cc b/utilities/write_batch_with_index/write_batch_with_index.cc index f20b7e5e3..d5a2f0351 100644 --- a/utilities/write_batch_with_index/write_batch_with_index.cc +++ b/utilities/write_batch_with_index/write_batch_with_index.cc @@ -3,15 +3,17 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). - #include "rocksdb/utilities/write_batch_with_index.h" +#include #include #include "db/column_family.h" #include "db/db_impl/db_impl.h" +#include "db/dbformat.h" #include "db/merge_context.h" #include "db/merge_helper.h" +#include "db/wide/wide_columns_helper.h" #include "memory/arena.h" #include "memtable/skiplist.h" #include "options/db_options.h" @@ -253,6 +255,14 @@ Status WriteBatchWithIndex::Rep::ReBuildIndex() { case kTypeRollbackXID: case kTypeNoop: break; + case kTypeColumnFamilyWideColumnEntity: + case kTypeWideColumnEntity: + found++; + if (!UpdateExistingEntryWithCfId(column_family_id, key, + kPutEntityRecord)) { + AddNewEntry(column_family_id); + } + break; default: return Status::Corruption( "unknown WriteBatch tag in ReBuildIndex", @@ -299,12 +309,20 @@ WBWIIterator* WriteBatchWithIndex::NewIterator( Iterator* WriteBatchWithIndex::NewIteratorWithBase( ColumnFamilyHandle* column_family, Iterator* base_iterator, const ReadOptions* read_options) { - auto wbwiii = - new WBWIIteratorImpl(GetColumnFamilyID(column_family), &(rep->skip_list), - &rep->write_batch, &rep->comparator); + WBWIIteratorImpl* wbwiii; + if (read_options != nullptr) { + wbwiii = new WBWIIteratorImpl( + GetColumnFamilyID(column_family), &(rep->skip_list), &rep->write_batch, + &rep->comparator, read_options->iterate_lower_bound, + read_options->iterate_upper_bound); + } else { + wbwiii = new WBWIIteratorImpl(GetColumnFamilyID(column_family), + &(rep->skip_list), &rep->write_batch, + &rep->comparator); + } + return new BaseDeltaIterator(column_family, base_iterator, wbwiii, - GetColumnFamilyUserComparator(column_family), - read_options); + GetColumnFamilyUserComparator(column_family)); } Iterator* WriteBatchWithIndex::NewIteratorWithBase(Iterator* base_iterator) { @@ -344,6 +362,22 @@ Status WriteBatchWithIndex::Put(ColumnFamilyHandle* column_family, return Status::NotSupported(); } +Status WriteBatchWithIndex::PutEntity(ColumnFamilyHandle* column_family, + const Slice& key, + const WideColumns& columns) { + assert(rep); + + rep->SetLastEntryOffset(); + + const Status s = rep->write_batch.PutEntity(column_family, key, columns); + + if (s.ok()) { + rep->AddOrUpdateIndex(column_family, key, kPutEntityRecord); + } + + return s; +} + Status WriteBatchWithIndex::Delete(ColumnFamilyHandle* column_family, const Slice& key) { rep->SetLastEntryOffset(); @@ -427,11 +461,12 @@ Status WriteBatchWithIndex::PutLogData(const Slice& blob) { void WriteBatchWithIndex::Clear() { rep->Clear(); } Status WriteBatchWithIndex::GetFromBatch(ColumnFamilyHandle* column_family, - const DBOptions& options, + const DBOptions& /* options */, const Slice& key, std::string* value) { + MergeContext merge_context; Status s; - WriteBatchWithIndexInternal wbwii(&options, column_family); - auto result = wbwii.GetFromBatch(this, key, value, &s); + auto result = WriteBatchWithIndexInternal::GetFromBatch( + this, column_family, key, &merge_context, value, &s); switch (result) { case WBWIIteratorImpl::kFound: @@ -500,61 +535,104 @@ Status WriteBatchWithIndex::GetFromBatchAndDB(DB* db, nullptr); } +void WriteBatchWithIndex::MergeAcrossBatchAndDB( + ColumnFamilyHandle* column_family, const Slice& key, + const PinnableWideColumns& existing, const MergeContext& merge_context, + PinnableSlice* value, Status* status) { + assert(value); + assert(status); + assert(status->ok() || status->IsNotFound()); + + std::string result_value; + + if (status->ok()) { + if (WideColumnsHelper::HasDefaultColumnOnly(existing.columns())) { + *status = WriteBatchWithIndexInternal::MergeKeyWithBaseValue( + column_family, key, MergeHelper::kPlainBaseValue, + WideColumnsHelper::GetDefaultColumn(existing.columns()), + merge_context, &result_value, + static_cast(nullptr)); + } else { + *status = WriteBatchWithIndexInternal::MergeKeyWithBaseValue( + column_family, key, MergeHelper::kWideBaseValue, existing.columns(), + merge_context, &result_value, + static_cast(nullptr)); + } + } else { + assert(status->IsNotFound()); + *status = WriteBatchWithIndexInternal::MergeKeyWithNoBaseValue( + column_family, key, merge_context, &result_value, + static_cast(nullptr)); + } + + if (status->ok()) { + value->Reset(); + *value->GetSelf() = std::move(result_value); + value->PinSelf(); + } +} + Status WriteBatchWithIndex::GetFromBatchAndDB( DB* db, const ReadOptions& read_options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* pinnable_val, ReadCallback* callback) { + assert(db); + assert(pinnable_val); + + if (!column_family) { + column_family = db->DefaultColumnFamily(); + } + const Comparator* const ucmp = rep->comparator.GetComparator(column_family); size_t ts_sz = ucmp ? ucmp->timestamp_size() : 0; if (ts_sz > 0 && !read_options.timestamp) { return Status::InvalidArgument("Must specify timestamp"); } - Status s; - WriteBatchWithIndexInternal wbwii(db, column_family); - // Since the lifetime of the WriteBatch is the same as that of the transaction // we cannot pin it as otherwise the returned value will not be available // after the transaction finishes. - std::string& batch_value = *pinnable_val->GetSelf(); - auto result = wbwii.GetFromBatch(this, key, &batch_value, &s); + MergeContext merge_context; + Status s; + + auto result = WriteBatchWithIndexInternal::GetFromBatch( + this, column_family, key, &merge_context, pinnable_val->GetSelf(), &s); if (result == WBWIIteratorImpl::kFound) { pinnable_val->PinSelf(); return s; - } else if (!s.ok() || result == WBWIIteratorImpl::kError) { + } + + if (!s.ok() || result == WBWIIteratorImpl::kError) { return s; - } else if (result == WBWIIteratorImpl::kDeleted) { + } + + if (result == WBWIIteratorImpl::kDeleted) { return Status::NotFound(); } - assert(result == WBWIIteratorImpl::kMergeInProgress || - result == WBWIIteratorImpl::kNotFound); // Did not find key in batch OR could not resolve Merges. Try DB. - if (!callback) { - s = db->Get(read_options, column_family, key, pinnable_val); + DBImpl::GetImplOptions get_impl_options; + get_impl_options.column_family = column_family; + + // Note: we have to retrieve all columns if we have to merge KVs from the + // batch and the DB; otherwise, the default column is sufficient. + PinnableWideColumns existing; + + if (result == WBWIIteratorImpl::kMergeInProgress) { + get_impl_options.columns = &existing; } else { - DBImpl::GetImplOptions get_impl_options; - get_impl_options.column_family = column_family; + assert(result == WBWIIteratorImpl::kNotFound); get_impl_options.value = pinnable_val; - get_impl_options.callback = callback; - s = static_cast_with_check(db->GetRootDB()) - ->GetImpl(read_options, key, get_impl_options); } - if (s.ok() || s.IsNotFound()) { // DB Get Succeeded - if (result == WBWIIteratorImpl::kMergeInProgress) { - // Merge result from DB with merges in Batch - std::string merge_result; - if (s.ok()) { - s = wbwii.MergeKey(key, pinnable_val, &merge_result); - } else { // Key not present in db (s.IsNotFound()) - s = wbwii.MergeKey(key, nullptr, &merge_result); - } - if (s.ok()) { - pinnable_val->Reset(); - *pinnable_val->GetSelf() = std::move(merge_result); - pinnable_val->PinSelf(); - } + get_impl_options.callback = callback; + s = static_cast_with_check(db->GetRootDB()) + ->GetImpl(read_options, key, get_impl_options); + + if (result == WBWIIteratorImpl::kMergeInProgress) { + if (s.ok() || s.IsNotFound()) { // DB lookup succeeded + MergeAcrossBatchAndDB(column_family, key, existing, merge_context, + pinnable_val, &s); } } @@ -573,6 +651,15 @@ void WriteBatchWithIndex::MultiGetFromBatchAndDB( DB* db, const ReadOptions& read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, Status* statuses, bool sorted_input, ReadCallback* callback) { + assert(db); + assert(keys); + assert(values); + assert(statuses); + + if (!column_family) { + column_family = db->DefaultColumnFamily(); + } + const Comparator* const ucmp = rep->comparator.GetComparator(column_family); size_t ts_sz = ucmp ? ucmp->timestamp_size() : 0; if (ts_sz > 0 && !read_options.timestamp) { @@ -582,79 +669,101 @@ void WriteBatchWithIndex::MultiGetFromBatchAndDB( return; } - WriteBatchWithIndexInternal wbwii(db, column_family); + struct MergeTuple { + MergeTuple(const Slice& _key, Status* _s, MergeContext&& _merge_context, + PinnableSlice* _value) + : key(_key), + s(_s), + merge_context(std::move(_merge_context)), + value(_value) { + assert(s); + assert(value); + } + + Slice key; + Status* s; + PinnableWideColumns existing; + MergeContext merge_context; + PinnableSlice* value; + }; + + autovector merges; + + autovector key_contexts; - autovector key_context; - autovector sorted_keys; - // To hold merges from the write batch - autovector, - MultiGetContext::MAX_BATCH_SIZE> - merges; // Since the lifetime of the WriteBatch is the same as that of the transaction // we cannot pin it as otherwise the returned value will not be available // after the transaction finishes. for (size_t i = 0; i < num_keys; ++i) { + const Slice& key = keys[i]; MergeContext merge_context; std::string batch_value; - Status* s = &statuses[i]; - PinnableSlice* pinnable_val = &values[i]; + Status* const s = &statuses[i]; + auto result = WriteBatchWithIndexInternal::GetFromBatch( + this, column_family, key, &merge_context, &batch_value, s); + + PinnableSlice* const pinnable_val = &values[i]; pinnable_val->Reset(); - auto result = - wbwii.GetFromBatch(this, keys[i], &merge_context, &batch_value, s); if (result == WBWIIteratorImpl::kFound) { *pinnable_val->GetSelf() = std::move(batch_value); pinnable_val->PinSelf(); continue; } + if (result == WBWIIteratorImpl::kDeleted) { *s = Status::NotFound(); continue; } + if (result == WBWIIteratorImpl::kError) { continue; } - assert(result == WBWIIteratorImpl::kMergeInProgress || - result == WBWIIteratorImpl::kNotFound); - key_context.emplace_back(column_family, keys[i], &values[i], - /* columns */ nullptr, /* timestamp */ nullptr, - &statuses[i]); - merges.emplace_back(result, std::move(merge_context)); + + // Note: we have to retrieve all columns if we have to merge KVs from the + // batch and the DB; otherwise, the default column is sufficient. + // The columns field will be populated by the loop below to prevent issues + // with dangling pointers. + if (result == WBWIIteratorImpl::kMergeInProgress) { + merges.emplace_back(key, s, std::move(merge_context), pinnable_val); + key_contexts.emplace_back(column_family, key, /* value */ nullptr, + /* columns */ nullptr, /* timestamp */ nullptr, + s); + continue; + } + + assert(result == WBWIIteratorImpl::kNotFound); + key_contexts.emplace_back(column_family, key, pinnable_val, + /* columns */ nullptr, + /* timestamp */ nullptr, s); } - for (KeyContext& key : key_context) { - sorted_keys.emplace_back(&key); + autovector sorted_keys; + sorted_keys.reserve(key_contexts.size()); + + size_t merges_idx = 0; + for (KeyContext& key_context : key_contexts) { + if (!key_context.value) { + assert(*key_context.key == merges[merges_idx].key); + + key_context.columns = &merges[merges_idx].existing; + ++merges_idx; + } + + sorted_keys.emplace_back(&key_context); } // Did not find key in batch OR could not resolve Merges. Try DB. static_cast_with_check(db->GetRootDB()) - ->PrepareMultiGetKeys(key_context.size(), sorted_input, &sorted_keys); + ->PrepareMultiGetKeys(sorted_keys.size(), sorted_input, &sorted_keys); static_cast_with_check(db->GetRootDB()) ->MultiGetWithCallback(read_options, column_family, callback, &sorted_keys); - for (auto iter = key_context.begin(); iter != key_context.end(); ++iter) { - KeyContext& key = *iter; - if (key.s->ok() || key.s->IsNotFound()) { // DB Get Succeeded - size_t index = iter - key_context.begin(); - std::pair& merge_result = - merges[index]; - if (merge_result.first == WBWIIteratorImpl::kMergeInProgress) { - std::string merged_value; - // Merge result from DB with merges in Batch - if (key.s->ok()) { - *key.s = wbwii.MergeKey(*key.key, iter->value, merge_result.second, - &merged_value); - } else { // Key not present in db (s.IsNotFound()) - *key.s = wbwii.MergeKey(*key.key, nullptr, merge_result.second, - &merged_value); - } - if (key.s->ok()) { - key.value->Reset(); - *key.value->GetSelf() = std::move(merged_value); - key.value->PinSelf(); - } - } + for (const auto& merge : merges) { + if (merge.s->ok() || merge.s->IsNotFound()) { // DB lookup succeeded + MergeAcrossBatchAndDB(column_family, merge.key, merge.existing, + merge.merge_context, merge.value, merge.s); } } } diff --git a/utilities/write_batch_with_index/write_batch_with_index_internal.cc b/utilities/write_batch_with_index/write_batch_with_index_internal.cc index ee4754f8d..2ae9fa86f 100644 --- a/utilities/write_batch_with_index/write_batch_with_index_internal.cc +++ b/utilities/write_batch_with_index/write_batch_with_index_internal.cc @@ -3,13 +3,14 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). - #include "utilities/write_batch_with_index/write_batch_with_index_internal.h" #include "db/column_family.h" #include "db/db_impl/db_impl.h" -#include "db/merge_context.h" #include "db/merge_helper.h" +#include "db/wide/wide_column_serialization.h" +#include "db/wide/wide_columns_helper.h" +#include "options/cf_options.h" #include "rocksdb/comparator.h" #include "rocksdb/db.h" #include "rocksdb/utilities/write_batch_with_index.h" @@ -21,19 +22,18 @@ namespace ROCKSDB_NAMESPACE { BaseDeltaIterator::BaseDeltaIterator(ColumnFamilyHandle* column_family, Iterator* base_iterator, WBWIIteratorImpl* delta_iterator, - const Comparator* comparator, - const ReadOptions* read_options) + const Comparator* comparator) : forward_(true), current_at_base_(true), equal_keys_(false), status_(Status::OK()), + column_family_(column_family), base_iterator_(base_iterator), delta_iterator_(delta_iterator), - comparator_(comparator), - iterate_upper_bound_(read_options ? read_options->iterate_upper_bound - : nullptr) { + comparator_(comparator) { + assert(base_iterator_); + assert(delta_iterator_); assert(comparator_); - wbwii_.reset(new WriteBatchWithIndexInternal(column_family)); } bool BaseDeltaIterator::Valid() const { @@ -148,33 +148,8 @@ Slice BaseDeltaIterator::key() const { : delta_iterator_->Entry().key; } -Slice BaseDeltaIterator::value() const { - if (current_at_base_) { - return base_iterator_->value(); - } else { - WriteEntry delta_entry = delta_iterator_->Entry(); - if (wbwii_->GetNumOperands() == 0) { - return delta_entry.value; - } else if (delta_entry.type == kDeleteRecord || - delta_entry.type == kSingleDeleteRecord) { - status_ = - wbwii_->MergeKey(delta_entry.key, nullptr, merge_result_.GetSelf()); - } else if (delta_entry.type == kPutRecord) { - status_ = wbwii_->MergeKey(delta_entry.key, &delta_entry.value, - merge_result_.GetSelf()); - } else if (delta_entry.type == kMergeRecord) { - if (equal_keys_) { - Slice base_value = base_iterator_->value(); - status_ = wbwii_->MergeKey(delta_entry.key, &base_value, - merge_result_.GetSelf()); - } else { - status_ = - wbwii_->MergeKey(delta_entry.key, nullptr, merge_result_.GetSelf()); - } - } - merge_result_.PinSelf(); - return merge_result_; - } +Slice BaseDeltaIterator::timestamp() const { + return current_at_base_ ? base_iterator_->timestamp() : Slice(); } Status BaseDeltaIterator::status() const { @@ -273,17 +248,126 @@ void BaseDeltaIterator::AdvanceBase() { bool BaseDeltaIterator::BaseValid() const { return base_iterator_->Valid(); } bool BaseDeltaIterator::DeltaValid() const { return delta_iterator_->Valid(); } + +void BaseDeltaIterator::ResetValueAndColumns() { + value_.clear(); + columns_.clear(); +} + +void BaseDeltaIterator::SetValueAndColumnsFromBase() { + assert(current_at_base_); + assert(BaseValid()); + assert(value_.empty()); + assert(columns_.empty()); + + value_ = base_iterator_->value(); + columns_ = base_iterator_->columns(); +} + +void BaseDeltaIterator::SetValueAndColumnsFromDelta() { + assert(!current_at_base_); + assert(DeltaValid()); + assert(value_.empty()); + assert(columns_.empty()); + + WriteEntry delta_entry = delta_iterator_->Entry(); + + if (merge_context_.GetNumOperands() == 0) { + if (delta_entry.type == kPutRecord) { + value_ = delta_entry.value; + columns_.emplace_back(kDefaultWideColumnName, value_); + } else if (delta_entry.type == kPutEntityRecord) { + Slice value_copy(delta_entry.value); + + status_ = WideColumnSerialization::Deserialize(value_copy, columns_); + if (!status_.ok()) { + return; + } + + if (WideColumnsHelper::HasDefaultColumn(columns_)) { + value_ = WideColumnsHelper::GetDefaultColumn(columns_); + } + } + + return; + } + + ValueType result_type = kTypeValue; + + if (delta_entry.type == kDeleteRecord || + delta_entry.type == kSingleDeleteRecord) { + status_ = WriteBatchWithIndexInternal::MergeKeyWithNoBaseValue( + column_family_, delta_entry.key, merge_context_, &merge_result_, + /* result_operand */ nullptr, &result_type); + } else if (delta_entry.type == kPutRecord) { + status_ = WriteBatchWithIndexInternal::MergeKeyWithBaseValue( + column_family_, delta_entry.key, MergeHelper::kPlainBaseValue, + delta_entry.value, merge_context_, &merge_result_, + /* result_operand */ nullptr, &result_type); + } else if (delta_entry.type == kPutEntityRecord) { + status_ = WriteBatchWithIndexInternal::MergeKeyWithBaseValue( + column_family_, delta_entry.key, MergeHelper::kWideBaseValue, + delta_entry.value, merge_context_, &merge_result_, + /* result_operand */ nullptr, &result_type); + } else if (delta_entry.type == kMergeRecord) { + if (equal_keys_) { + if (WideColumnsHelper::HasDefaultColumnOnly(base_iterator_->columns())) { + status_ = WriteBatchWithIndexInternal::MergeKeyWithBaseValue( + column_family_, delta_entry.key, MergeHelper::kPlainBaseValue, + base_iterator_->value(), merge_context_, &merge_result_, + /* result_operand */ nullptr, &result_type); + } else { + status_ = WriteBatchWithIndexInternal::MergeKeyWithBaseValue( + column_family_, delta_entry.key, MergeHelper::kWideBaseValue, + base_iterator_->columns(), merge_context_, &merge_result_, + /* result_operand */ nullptr, &result_type); + } + } else { + status_ = WriteBatchWithIndexInternal::MergeKeyWithNoBaseValue( + column_family_, delta_entry.key, merge_context_, &merge_result_, + /* result_operand */ nullptr, &result_type); + } + } else { + status_ = Status::NotSupported("Unsupported entry type for merge"); + } + + if (!status_.ok()) { + return; + } + + if (result_type == kTypeWideColumnEntity) { + Slice entity(merge_result_); + + status_ = WideColumnSerialization::Deserialize(entity, columns_); + if (!status_.ok()) { + return; + } + + if (WideColumnsHelper::HasDefaultColumn(columns_)) { + value_ = WideColumnsHelper::GetDefaultColumn(columns_); + } + + return; + } + + assert(result_type == kTypeValue); + + value_ = merge_result_; + columns_.emplace_back(kDefaultWideColumnName, value_); +} + void BaseDeltaIterator::UpdateCurrent() { // Suppress false positive clang analyzer warnings. #ifndef __clang_analyzer__ status_ = Status::OK(); + ResetValueAndColumns(); + while (true) { auto delta_result = WBWIIteratorImpl::kNotFound; WriteEntry delta_entry; if (DeltaValid()) { assert(delta_iterator_->status().ok()); - delta_result = - delta_iterator_->FindLatestUpdate(wbwii_->GetMergeContext()); + delta_result = delta_iterator_->FindLatestUpdate(&merge_context_); delta_entry = delta_iterator_->Entry(); } else if (!delta_iterator_->status().ok()) { // Expose the error status and stop. @@ -303,24 +387,18 @@ void BaseDeltaIterator::UpdateCurrent() { // Finished return; } - if (iterate_upper_bound_) { - if (comparator_->CompareWithoutTimestamp( - delta_entry.key, /*a_has_ts=*/false, *iterate_upper_bound_, - /*b_has_ts=*/false) >= 0) { - // out of upper bound -> finished. - return; - } - } if (delta_result == WBWIIteratorImpl::kDeleted && - wbwii_->GetNumOperands() == 0) { + merge_context_.GetNumOperands() == 0) { AdvanceDelta(); } else { current_at_base_ = false; + SetValueAndColumnsFromDelta(); return; } } else if (!DeltaValid()) { // Delta has finished. current_at_base_ = true; + SetValueAndColumnsFromBase(); return; } else { int compare = @@ -332,8 +410,9 @@ void BaseDeltaIterator::UpdateCurrent() { equal_keys_ = true; } if (delta_result != WBWIIteratorImpl::kDeleted || - wbwii_->GetNumOperands() > 0) { + merge_context_.GetNumOperands() > 0) { current_at_base_ = false; + SetValueAndColumnsFromDelta(); return; } // Delta is less advanced and is delete. @@ -343,6 +422,7 @@ void BaseDeltaIterator::UpdateCurrent() { } } else { current_at_base_ = true; + SetValueAndColumnsFromBase(); return; } } @@ -435,6 +515,8 @@ WBWIIteratorImpl::Result WBWIIteratorImpl::FindLatestUpdate( break; // ignore case kXIDRecord: break; // ignore + case kPutEntityRecord: + return WBWIIteratorImpl::kFound; default: return WBWIIteratorImpl::kError; } // end switch statement @@ -453,10 +535,10 @@ WBWIIteratorImpl::Result WBWIIteratorImpl::FindLatestUpdate( } Status ReadableWriteBatch::GetEntryFromDataOffset(size_t data_offset, - WriteType* type, Slice* Key, + WriteType* type, Slice* key, Slice* value, Slice* blob, Slice* xid) const { - if (type == nullptr || Key == nullptr || value == nullptr || + if (type == nullptr || key == nullptr || value == nullptr || blob == nullptr || xid == nullptr) { return Status::InvalidArgument("Output parameters cannot be null"); } @@ -472,7 +554,7 @@ Status ReadableWriteBatch::GetEntryFromDataOffset(size_t data_offset, Slice input = Slice(rep_.data() + data_offset, rep_.size() - data_offset); char tag; uint32_t column_family; - Status s = ReadRecordFromWriteBatch(&input, &tag, &column_family, Key, value, + Status s = ReadRecordFromWriteBatch(&input, &tag, &column_family, key, value, blob, xid); if (!s.ok()) { return s; @@ -511,6 +593,11 @@ Status ReadableWriteBatch::GetEntryFromDataOffset(size_t data_offset, case kTypeRollbackXID: *type = kXIDRecord; break; + case kTypeColumnFamilyWideColumnEntity: + case kTypeWideColumnEntity: { + *type = kPutEntityRecord; + break; + } default: return Status::Corruption("unknown WriteBatch tag ", std::to_string(static_cast(tag))); @@ -610,9 +697,9 @@ WriteEntry WBWIIteratorImpl::Entry() const { auto s = write_batch_->GetEntryFromDataOffset( iter_entry->offset, &ret.type, &ret.key, &ret.value, &blob, &xid); assert(s.ok()); - assert(ret.type == kPutRecord || ret.type == kDeleteRecord || - ret.type == kSingleDeleteRecord || ret.type == kDeleteRangeRecord || - ret.type == kMergeRecord); + assert(ret.type == kPutRecord || ret.type == kPutEntityRecord || + ret.type == kDeleteRecord || ret.type == kSingleDeleteRecord || + ret.type == kDeleteRangeRecord || ret.type == kMergeRecord); // Make sure entry.key does not include user-defined timestamp. const Comparator* const ucmp = comparator_->GetComparator(column_family_id_); size_t ts_sz = ucmp->timestamp_size(); @@ -630,81 +717,36 @@ bool WBWIIteratorImpl::MatchesKey(uint32_t cf_id, const Slice& key) { } } -WriteBatchWithIndexInternal::WriteBatchWithIndexInternal( - ColumnFamilyHandle* column_family) - : db_(nullptr), db_options_(nullptr), column_family_(column_family) {} - -WriteBatchWithIndexInternal::WriteBatchWithIndexInternal( - DB* db, ColumnFamilyHandle* column_family) - : db_(db), db_options_(nullptr), column_family_(column_family) { - if (db_ != nullptr && column_family_ == nullptr) { - column_family_ = db_->DefaultColumnFamily(); - } -} - -WriteBatchWithIndexInternal::WriteBatchWithIndexInternal( - const DBOptions* db_options, ColumnFamilyHandle* column_family) - : db_(nullptr), db_options_(db_options), column_family_(column_family) {} - -Status WriteBatchWithIndexInternal::MergeKey(const Slice& key, - const Slice* value, - const MergeContext& context, - std::string* result) const { - if (column_family_ != nullptr) { - auto cfh = static_cast_with_check(column_family_); - const auto merge_operator = cfh->cfd()->ioptions()->merge_operator.get(); - if (merge_operator == nullptr) { - return Status::InvalidArgument( - "Merge_operator must be set for column_family"); - } else if (db_ != nullptr) { - const ImmutableDBOptions& immutable_db_options = - static_cast_with_check(db_->GetRootDB()) - ->immutable_db_options(); - Statistics* statistics = immutable_db_options.statistics.get(); - Logger* logger = immutable_db_options.info_log.get(); - SystemClock* clock = immutable_db_options.clock; - // `op_failure_scope` (an output parameter) is not provided (set to - // nullptr) since a failure must be propagated regardless of its value. - return MergeHelper::TimedFullMerge( - merge_operator, key, value, context.GetOperands(), result, logger, - statistics, clock, /* result_operand */ nullptr, - /* update_num_ops_stats */ false, - /* op_failure_scope */ nullptr); - } else if (db_options_ != nullptr) { - Statistics* statistics = db_options_->statistics.get(); - Env* env = db_options_->env; - Logger* logger = db_options_->info_log.get(); - SystemClock* clock = env->GetSystemClock().get(); - // `op_failure_scope` (an output parameter) is not provided (set to - // nullptr) since a failure must be propagated regardless of its value. - return MergeHelper::TimedFullMerge( - merge_operator, key, value, context.GetOperands(), result, logger, - statistics, clock, /* result_operand */ nullptr, - /* update_num_ops_stats */ false, - /* op_failure_scope */ nullptr); - } else { - const auto cf_opts = cfh->cfd()->ioptions(); - // `op_failure_scope` (an output parameter) is not provided (set to - // nullptr) since a failure must be propagated regardless of its value. - return MergeHelper::TimedFullMerge( - merge_operator, key, value, context.GetOperands(), result, - cf_opts->logger, cf_opts->stats, cf_opts->clock, - /* result_operand */ nullptr, /* update_num_ops_stats */ false, - /* op_failure_scope */ nullptr); - } - } else { - return Status::InvalidArgument("Must provide a column_family"); +Status WriteBatchWithIndexInternal::CheckAndGetImmutableOptions( + ColumnFamilyHandle* column_family, const ImmutableOptions** ioptions) { + assert(ioptions); + assert(!*ioptions); + + if (!column_family) { + return Status::InvalidArgument("Must provide a column family"); } + + const auto& iopts = GetImmutableOptions(column_family); + + const auto* merge_operator = iopts.merge_operator.get(); + if (!merge_operator) { + return Status::InvalidArgument( + "Merge operator must be set for column family"); + } + + *ioptions = &iopts; + + return Status::OK(); } WBWIIteratorImpl::Result WriteBatchWithIndexInternal::GetFromBatch( - WriteBatchWithIndex* batch, const Slice& key, MergeContext* context, - std::string* value, Status* s) { + WriteBatchWithIndex* batch, ColumnFamilyHandle* column_family, + const Slice& key, MergeContext* context, std::string* value, Status* s) { *s = Status::OK(); std::unique_ptr iter( static_cast_with_check( - batch->NewIterator(column_family_))); + batch->NewIterator(column_family))); // Search the iterator for this key, and updates/merges to it. iter->Seek(key); @@ -715,19 +757,41 @@ WBWIIteratorImpl::Result WriteBatchWithIndexInternal::GetFromBatch( return result; } else if (result == WBWIIteratorImpl::kNotFound) { return result; - } else if (result == WBWIIteratorImpl::Result::kFound) { // PUT - Slice entry_value = iter->Entry().value; + } else if (result == WBWIIteratorImpl::Result::kFound) { // Put/PutEntity + WriteEntry entry = iter->Entry(); + Slice entry_value = entry.value; if (context->GetNumOperands() > 0) { - *s = MergeKey(key, &entry_value, *context, value); + if (entry.type == kPutRecord) { + *s = MergeKeyWithBaseValue( + column_family, key, MergeHelper::kPlainBaseValue, entry_value, + *context, value, static_cast(nullptr)); + } else { + assert(entry.type == kPutEntityRecord); + + *s = MergeKeyWithBaseValue( + column_family, key, MergeHelper::kWideBaseValue, entry_value, + *context, value, static_cast(nullptr)); + } if (!s->ok()) { result = WBWIIteratorImpl::Result::kError; } } else { - value->assign(entry_value.data(), entry_value.size()); + if (entry.type == kPutRecord) { + value->assign(entry_value.data(), entry_value.size()); + } else { + assert(entry.type == kPutEntityRecord); + Slice value_of_default; + *s = WideColumnSerialization::GetValueOfDefaultColumn(entry_value, + value_of_default); + if (s->ok()) { + value->assign(value_of_default.data(), value_of_default.size()); + } + } } } else if (result == WBWIIteratorImpl::kDeleted) { if (context->GetNumOperands() > 0) { - *s = MergeKey(key, nullptr, *context, value); + *s = MergeKeyWithNoBaseValue(column_family, key, *context, value, + static_cast(nullptr)); if (s->ok()) { result = WBWIIteratorImpl::Result::kFound; } else { @@ -739,4 +803,3 @@ WBWIIteratorImpl::Result WriteBatchWithIndexInternal::GetFromBatch( } } // namespace ROCKSDB_NAMESPACE - diff --git a/utilities/write_batch_with_index/write_batch_with_index_internal.h b/utilities/write_batch_with_index/write_batch_with_index_internal.h index 031d72889..e1c97a9eb 100644 --- a/utilities/write_batch_with_index/write_batch_with_index_internal.h +++ b/utilities/write_batch_with_index/write_batch_with_index_internal.h @@ -4,12 +4,13 @@ // (found in the LICENSE.Apache file in the root directory). #pragma once - #include #include #include +#include "db/dbformat.h" #include "db/merge_context.h" +#include "db/merge_helper.h" #include "memtable/skiplist.h" #include "options/db_options.h" #include "port/port.h" @@ -21,10 +22,9 @@ namespace ROCKSDB_NAMESPACE { -class MergeContext; class WBWIIteratorImpl; -class WriteBatchWithIndexInternal; struct Options; +struct ImmutableOptions; // when direction == forward // * current_at_base_ <=> base_iterator > delta_iterator @@ -36,8 +36,7 @@ class BaseDeltaIterator : public Iterator { public: BaseDeltaIterator(ColumnFamilyHandle* column_family, Iterator* base_iterator, WBWIIteratorImpl* delta_iterator, - const Comparator* comparator, - const ReadOptions* read_options = nullptr); + const Comparator* comparator); ~BaseDeltaIterator() override {} @@ -49,7 +48,9 @@ class BaseDeltaIterator : public Iterator { void Next() override; void Prev() override; Slice key() const override; - Slice value() const override; + Slice value() const override { return value_; } + const WideColumns& columns() const override { return columns_; } + Slice timestamp() const override; Status status() const override; void Invalidate(Status s); @@ -60,18 +61,23 @@ class BaseDeltaIterator : public Iterator { void AdvanceBase(); bool BaseValid() const; bool DeltaValid() const; + void ResetValueAndColumns(); + void SetValueAndColumnsFromBase(); + void SetValueAndColumnsFromDelta(); void UpdateCurrent(); - std::unique_ptr wbwii_; bool forward_; bool current_at_base_; bool equal_keys_; - mutable Status status_; + Status status_; + ColumnFamilyHandle* column_family_; std::unique_ptr base_iterator_; std::unique_ptr delta_iterator_; const Comparator* comparator_; // not owned - const Slice* iterate_upper_bound_; - mutable PinnableSlice merge_result_; + MergeContext merge_context_; + std::string merge_result_; + Slice value_; + WideColumns columns_; }; // Key used by skip list, as the binary searchable index of WriteBatchWithIndex. @@ -143,7 +149,7 @@ class ReadableWriteBatch : public WriteBatch { default_cf_ts_sz) {} // Retrieve some information from a write entry in the write batch, given // the start offset of the write entry. - Status GetEntryFromDataOffset(size_t data_offset, WriteType* type, Slice* Key, + Status GetEntryFromDataOffset(size_t data_offset, WriteType* type, Slice* key, Slice* value, Slice* blob, Slice* xid) const; }; @@ -196,59 +202,107 @@ class WBWIIteratorImpl : public WBWIIterator { WBWIIteratorImpl(uint32_t column_family_id, WriteBatchEntrySkipList* skip_list, const ReadableWriteBatch* write_batch, - WriteBatchEntryComparator* comparator) + WriteBatchEntryComparator* comparator, + const Slice* iterate_lower_bound = nullptr, + const Slice* iterate_upper_bound = nullptr) : column_family_id_(column_family_id), skip_list_iter_(skip_list), write_batch_(write_batch), - comparator_(comparator) {} + comparator_(comparator), + iterate_lower_bound_(iterate_lower_bound), + iterate_upper_bound_(iterate_upper_bound) {} ~WBWIIteratorImpl() override {} bool Valid() const override { - if (!skip_list_iter_.Valid()) { - return false; - } - const WriteBatchIndexEntry* iter_entry = skip_list_iter_.key(); - return (iter_entry != nullptr && - iter_entry->column_family == column_family_id_); + return !out_of_bound_ && ValidRegardlessOfBoundLimit(); } void SeekToFirst() override { - WriteBatchIndexEntry search_entry( - nullptr /* search_key */, column_family_id_, - true /* is_forward_direction */, true /* is_seek_to_first */); - skip_list_iter_.Seek(&search_entry); + if (iterate_lower_bound_ != nullptr) { + WriteBatchIndexEntry search_entry( + iterate_lower_bound_ /* search_key */, column_family_id_, + true /* is_forward_direction */, false /* is_seek_to_first */); + skip_list_iter_.Seek(&search_entry); + } else { + WriteBatchIndexEntry search_entry( + nullptr /* search_key */, column_family_id_, + true /* is_forward_direction */, true /* is_seek_to_first */); + skip_list_iter_.Seek(&search_entry); + } + + if (ValidRegardlessOfBoundLimit()) { + out_of_bound_ = TestOutOfBound(); + } } void SeekToLast() override { - WriteBatchIndexEntry search_entry( - nullptr /* search_key */, column_family_id_ + 1, - true /* is_forward_direction */, true /* is_seek_to_first */); + WriteBatchIndexEntry search_entry = + (iterate_upper_bound_ != nullptr) + ? WriteBatchIndexEntry( + iterate_upper_bound_ /* search_key */, column_family_id_, + true /* is_forward_direction */, false /* is_seek_to_first */) + : WriteBatchIndexEntry( + nullptr /* search_key */, column_family_id_ + 1, + true /* is_forward_direction */, true /* is_seek_to_first */); + skip_list_iter_.Seek(&search_entry); if (!skip_list_iter_.Valid()) { skip_list_iter_.SeekToLast(); } else { skip_list_iter_.Prev(); } + + if (ValidRegardlessOfBoundLimit()) { + out_of_bound_ = TestOutOfBound(); + } } void Seek(const Slice& key) override { + if (BeforeLowerBound(&key)) { // cap to prevent out of bound + SeekToFirst(); + return; + } + WriteBatchIndexEntry search_entry(&key, column_family_id_, true /* is_forward_direction */, false /* is_seek_to_first */); skip_list_iter_.Seek(&search_entry); + + if (ValidRegardlessOfBoundLimit()) { + out_of_bound_ = TestOutOfBound(); + } } void SeekForPrev(const Slice& key) override { + if (AtOrAfterUpperBound(&key)) { // cap to prevent out of bound + SeekToLast(); + return; + } + WriteBatchIndexEntry search_entry(&key, column_family_id_, false /* is_forward_direction */, false /* is_seek_to_first */); skip_list_iter_.SeekForPrev(&search_entry); + + if (ValidRegardlessOfBoundLimit()) { + out_of_bound_ = TestOutOfBound(); + } } - void Next() override { skip_list_iter_.Next(); } + void Next() override { + skip_list_iter_.Next(); + if (ValidRegardlessOfBoundLimit()) { + out_of_bound_ = TestOutOfBound(); + } + } - void Prev() override { skip_list_iter_.Prev(); } + void Prev() override { + skip_list_iter_.Prev(); + if (ValidRegardlessOfBoundLimit()) { + out_of_bound_ = TestOutOfBound(); + } + } WriteEntry Entry() const override; @@ -269,12 +323,12 @@ class WBWIIteratorImpl : public WBWIIterator { // Moves the iterator to first entry of the next key. void NextKey(); - // Moves the iterator to the Update (Put or Delete) for the current key - // If there are no Put/Delete, the Iterator will point to the first entry for - // this key - // @return kFound if a Put was found for the key + // Moves the iterator to the Update (Put, PutEntity or Delete) for the current + // key. If there is no Put/PutEntity/Delete, the Iterator will point to the + // first entry for this key. + // @return kFound if a Put/PutEntity was found for the key // @return kDeleted if a delete was found for the key - // @return kMergeInProgress if only merges were fouund for the key + // @return kMergeInProgress if only merges were found for the key // @return kError if an unsupported operation was found for the key // @return kNotFound if no operations were found for this key // @@ -289,6 +343,45 @@ class WBWIIteratorImpl : public WBWIIterator { WriteBatchEntrySkipList::Iterator skip_list_iter_; const ReadableWriteBatch* write_batch_; WriteBatchEntryComparator* comparator_; + const Slice* iterate_lower_bound_; + const Slice* iterate_upper_bound_; + bool out_of_bound_ = false; + + bool TestOutOfBound() const { + const Slice& curKey = Entry().key; + return AtOrAfterUpperBound(&curKey) || BeforeLowerBound(&curKey); + } + + bool ValidRegardlessOfBoundLimit() const { + if (!skip_list_iter_.Valid()) { + return false; + } + const WriteBatchIndexEntry* iter_entry = skip_list_iter_.key(); + return iter_entry != nullptr && + iter_entry->column_family == column_family_id_; + } + + bool AtOrAfterUpperBound(const Slice* k) const { + if (iterate_upper_bound_ == nullptr) { + return false; + } + + return comparator_->GetComparator(column_family_id_) + ->CompareWithoutTimestamp(*k, /*a_has_ts=*/false, + *iterate_upper_bound_, + /*b_has_ts=*/false) >= 0; + } + + bool BeforeLowerBound(const Slice* k) const { + if (iterate_lower_bound_ == nullptr) { + return false; + } + + return comparator_->GetComparator(column_family_id_) + ->CompareWithoutTimestamp(*k, /*a_has_ts=*/false, + *iterate_lower_bound_, + /*b_has_ts=*/false) < 0; + } }; class WriteBatchWithIndexInternal { @@ -296,14 +389,52 @@ class WriteBatchWithIndexInternal { static const Comparator* GetUserComparator(const WriteBatchWithIndex& wbwi, uint32_t cf_id); - // For GetFromBatchAndDB or similar - explicit WriteBatchWithIndexInternal(DB* db, - ColumnFamilyHandle* column_family); - // For GetFromBatchAndDB or similar - explicit WriteBatchWithIndexInternal(ColumnFamilyHandle* column_family); - // For GetFromBatch or similar - explicit WriteBatchWithIndexInternal(const DBOptions* db_options, - ColumnFamilyHandle* column_family); + template + static Status MergeKeyWithNoBaseValue(ColumnFamilyHandle* column_family, + const Slice& key, + const MergeContext& context, + ResultTs... results) { + const ImmutableOptions* ioptions = nullptr; + + const Status s = CheckAndGetImmutableOptions(column_family, &ioptions); + if (!s.ok()) { + return s; + } + + assert(ioptions); + + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its value. + return MergeHelper::TimedFullMerge( + ioptions->merge_operator.get(), key, MergeHelper::kNoBaseValue, + context.GetOperands(), ioptions->logger, ioptions->stats, + ioptions->clock, /* update_num_ops_stats */ false, + /* op_failure_scope */ nullptr, results...); + } + + template + static Status MergeKeyWithBaseValue(ColumnFamilyHandle* column_family, + const Slice& key, const BaseTag& base_tag, + const BaseT& value, + const MergeContext& context, + ResultTs... results) { + const ImmutableOptions* ioptions = nullptr; + + const Status s = CheckAndGetImmutableOptions(column_family, &ioptions); + if (!s.ok()) { + return s; + } + + assert(ioptions); + + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its value. + return MergeHelper::TimedFullMerge( + ioptions->merge_operator.get(), key, base_tag, value, + context.GetOperands(), ioptions->logger, ioptions->stats, + ioptions->clock, /* update_num_ops_stats */ false, + /* op_failure_scope */ nullptr, results...); + } // If batch contains a value for key, store it in *value and return kFound. // If batch contains a deletion for key, return Deleted. @@ -313,30 +444,14 @@ class WriteBatchWithIndexInternal { // and return kMergeInProgress // If batch does not contain this key, return kNotFound // Else, return kError on error with error Status stored in *s. - WBWIIteratorImpl::Result GetFromBatch(WriteBatchWithIndex* batch, - const Slice& key, std::string* value, - Status* s) { - return GetFromBatch(batch, key, &merge_context_, value, s); - } - WBWIIteratorImpl::Result GetFromBatch(WriteBatchWithIndex* batch, - const Slice& key, - MergeContext* merge_context, - std::string* value, Status* s); - Status MergeKey(const Slice& key, const Slice* value, - std::string* result) const { - return MergeKey(key, value, merge_context_, result); - } - Status MergeKey(const Slice& key, const Slice* value, - const MergeContext& context, std::string* result) const; - size_t GetNumOperands() const { return merge_context_.GetNumOperands(); } - MergeContext* GetMergeContext() { return &merge_context_; } - Slice GetOperand(int index) const { return merge_context_.GetOperand(index); } + static WBWIIteratorImpl::Result GetFromBatch( + WriteBatchWithIndex* batch, ColumnFamilyHandle* column_family, + const Slice& key, MergeContext* merge_context, std::string* value, + Status* s); private: - DB* db_; - const DBOptions* db_options_; - ColumnFamilyHandle* column_family_; - MergeContext merge_context_; + static Status CheckAndGetImmutableOptions(ColumnFamilyHandle* column_family, + const ImmutableOptions** ioptions); }; } // namespace ROCKSDB_NAMESPACE diff --git a/utilities/write_batch_with_index/write_batch_with_index_test.cc b/utilities/write_batch_with_index/write_batch_with_index_test.cc index 7330c82ca..90438ff2e 100644 --- a/utilities/write_batch_with_index/write_batch_with_index_test.cc +++ b/utilities/write_batch_with_index/write_batch_with_index_test.cc @@ -7,7 +7,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. - #include "rocksdb/utilities/write_batch_with_index.h" #include @@ -82,37 +81,83 @@ using KVMap = std::map; class KVIter : public Iterator { public: explicit KVIter(const KVMap* map) : map_(map), iter_(map_->end()) {} + bool Valid() const override { return iter_ != map_->end(); } - void SeekToFirst() override { iter_ = map_->begin(); } + + void SeekToFirst() override { + iter_ = map_->begin(); + + if (Valid()) { + Update(); + } + } + void SeekToLast() override { if (map_->empty()) { iter_ = map_->end(); } else { iter_ = map_->find(map_->rbegin()->first); } + + if (Valid()) { + Update(); + } } + void Seek(const Slice& k) override { iter_ = map_->lower_bound(k.ToString()); + + if (Valid()) { + Update(); + } } + void SeekForPrev(const Slice& k) override { iter_ = map_->upper_bound(k.ToString()); Prev(); + + if (Valid()) { + Update(); + } } - void Next() override { ++iter_; } + + void Next() override { + ++iter_; + + if (Valid()) { + Update(); + } + } + void Prev() override { if (iter_ == map_->begin()) { iter_ = map_->end(); return; } --iter_; + + if (Valid()) { + Update(); + } } + Slice key() const override { return iter_->first; } - Slice value() const override { return iter_->second; } + Slice value() const override { return value_; } + const WideColumns& columns() const override { return columns_; } Status status() const override { return Status::OK(); } private: + void Update() { + assert(Valid()); + + value_ = iter_->second; + columns_ = WideColumns{{kDefaultWideColumnName, value_}}; + } + const KVMap* const map_; KVMap::const_iterator iter_; + Slice value_; + WideColumns columns_; }; static std::string PrintContents(WriteBatchWithIndex* batch, @@ -261,14 +306,20 @@ class WBWIBaseTest : public testing::Test { std::string result; for (size_t i = 0; i < key.size(); i++) { if (key[i] == 'd') { - batch_->Delete(cf, key); + EXPECT_OK(batch_->Delete(cf, key)); result = ""; } else if (key[i] == 'p') { result = key + std::to_string(i); - batch_->Put(cf, key, result); + EXPECT_OK(batch_->Put(cf, key, result)); + } else if (key[i] == 'e') { + const std::string suffix = std::to_string(i); + result = key + suffix; + const WideColumns columns{{kDefaultWideColumnName, result}, + {key, suffix}}; + EXPECT_OK(batch_->PutEntity(cf, key, columns)); } else if (key[i] == 'm') { std::string value = key + std::to_string(i); - batch_->Merge(cf, key, value); + EXPECT_OK(batch_->Merge(cf, key, value)); if (result.empty()) { result = value; } else { @@ -1243,7 +1294,7 @@ TEST_F(WBWIOverwriteTest, TestGetFromBatchMerge2) { s = batch_->GetFromBatch(column_family, options_, "X", &value); ASSERT_TRUE(s.IsNotFound()); - batch_->Merge(column_family, "X", "ddd"); + ASSERT_OK(batch_->Merge(column_family, "X", "ddd")); ASSERT_OK(batch_->GetFromBatch(column_family, options_, "X", &value)); ASSERT_EQ("ddd", value); } @@ -1641,6 +1692,104 @@ TEST_P(WriteBatchWithIndexTest, TestNewIteratorWithBaseFromWbwi) { ASSERT_OK(iter->status()); } +TEST_P(WriteBatchWithIndexTest, TestBoundsCheckingInDeltaIterator) { + Status s = OpenDB(); + ASSERT_OK(s); + + KVMap empty_map; + + // writes that should be observed by BaseDeltaIterator::delta_iterator_ + ASSERT_OK(batch_->Put("a", "aa")); + ASSERT_OK(batch_->Put("b", "bb")); + ASSERT_OK(batch_->Put("c", "cc")); + + ReadOptions ro; + + auto check_only_b_is_visible = [&]() { + std::unique_ptr iter(batch_->NewIteratorWithBase( + db_->DefaultColumnFamily(), new KVIter(&empty_map), &ro)); + + // move to the lower bound + iter->SeekToFirst(); + ASSERT_EQ("b", iter->key()); + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + + // move to the upper bound + iter->SeekToLast(); + ASSERT_EQ("b", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + // test bounds checking in Seek and SeekForPrev + iter->Seek(Slice("a")); + ASSERT_EQ("b", iter->key()); + iter->Seek(Slice("b")); + ASSERT_EQ("b", iter->key()); + iter->Seek(Slice("c")); + ASSERT_FALSE(iter->Valid()); + + iter->SeekForPrev(Slice("c")); + ASSERT_EQ("b", iter->key()); + iter->SeekForPrev(Slice("b")); + ASSERT_EQ("b", iter->key()); + iter->SeekForPrev(Slice("a")); + ASSERT_FALSE(iter->Valid()); + + iter->SeekForPrev( + Slice("a.1")); // a non-existent key that is smaller than "b" + ASSERT_FALSE(iter->Valid()); + + iter->Seek(Slice("b.1")); // a non-existent key that is greater than "b" + ASSERT_FALSE(iter->Valid()); + + delete ro.iterate_lower_bound; + delete ro.iterate_upper_bound; + }; + + ro.iterate_lower_bound = new Slice("b"); + ro.iterate_upper_bound = new Slice("c"); + check_only_b_is_visible(); + + ro.iterate_lower_bound = new Slice("a.1"); + ro.iterate_upper_bound = new Slice("c"); + check_only_b_is_visible(); + + ro.iterate_lower_bound = new Slice("b"); + ro.iterate_upper_bound = new Slice("b.2"); + check_only_b_is_visible(); +} + +TEST_P(WriteBatchWithIndexTest, + TestBoundsCheckingInSeekToFirstAndLastOfDeltaIterator) { + Status s = OpenDB(); + ASSERT_OK(s); + KVMap empty_map; + // writes that should be observed by BaseDeltaIterator::delta_iterator_ + ASSERT_OK(batch_->Put("c", "cc")); + + ReadOptions ro; + auto check_nothing_visible = [&]() { + std::unique_ptr iter(batch_->NewIteratorWithBase( + db_->DefaultColumnFamily(), new KVIter(&empty_map), &ro)); + iter->SeekToFirst(); + ASSERT_FALSE(iter->Valid()); + iter->SeekToLast(); + ASSERT_FALSE(iter->Valid()); + + delete ro.iterate_lower_bound; + delete ro.iterate_upper_bound; + }; + + ro.iterate_lower_bound = new Slice("b"); + ro.iterate_upper_bound = new Slice("c"); + check_nothing_visible(); + + ro.iterate_lower_bound = new Slice("d"); + ro.iterate_upper_bound = new Slice("e"); + check_nothing_visible(); +} + TEST_P(WriteBatchWithIndexTest, SavePointTest) { ColumnFamilyHandleImplDummy cf1(1, BytewiseComparator()); KVMap empty_map; @@ -2100,8 +2249,8 @@ TEST_P(WriteBatchWithIndexTest, GetFromBatchAfterMerge) { ASSERT_OK(OpenDB()); ASSERT_OK(db_->Put(write_opts_, "o", "aa")); - batch_->Merge("o", "bb"); // Merging bb under key "o" - batch_->Merge("m", "cc"); // Merging bc under key "m" + ASSERT_OK(batch_->Merge("o", "bb")); // Merging bb under key "o" + ASSERT_OK(batch_->Merge("m", "cc")); // Merging bc under key "m" s = batch_->GetFromBatch(options_, "m", &value); ASSERT_EQ(s.code(), Status::Code::kMergeInProgress); s = batch_->GetFromBatch(options_, "o", &value); @@ -2248,6 +2397,8 @@ TEST_F(WBWIOverwriteTest, TestBadMergeOperator) { } TEST_P(WriteBatchWithIndexTest, ColumnFamilyWithTimestamp) { + ASSERT_OK(OpenDB()); + ColumnFamilyHandleImplDummy cf2(2, test::BytewiseComparatorWithU64TsWrapper()); @@ -2263,10 +2414,9 @@ TEST_P(WriteBatchWithIndexTest, ColumnFamilyWithTimestamp) { .IsInvalidArgument()); { std::string value; - ASSERT_TRUE(batch_ - ->GetFromBatchAndDB( - /*db=*/nullptr, ReadOptions(), &cf2, "key", &value) - .IsInvalidArgument()); + ASSERT_TRUE( + batch_->GetFromBatchAndDB(db_, ReadOptions(), &cf2, "key", &value) + .IsInvalidArgument()); } { constexpr size_t num_keys = 2; @@ -2275,8 +2425,8 @@ TEST_P(WriteBatchWithIndexTest, ColumnFamilyWithTimestamp) { {PinnableSlice(), PinnableSlice()}}; std::array statuses{{Status(), Status()}}; constexpr bool sorted_input = false; - batch_->MultiGetFromBatchAndDB(/*db=*/nullptr, ReadOptions(), &cf2, - num_keys, keys.data(), pinnable_vals.data(), + batch_->MultiGetFromBatchAndDB(db_, ReadOptions(), &cf2, num_keys, + keys.data(), pinnable_vals.data(), statuses.data(), sorted_input); for (const auto& s : statuses) { ASSERT_TRUE(s.IsInvalidArgument()); @@ -2398,6 +2548,224 @@ TEST_P(WriteBatchWithIndexTest, IndexNoTs) { } } +TEST_P(WriteBatchWithIndexTest, WideColumnsBatchOnly) { + // Tests for the case when there's no need to consult the underlying DB during + // queries, i.e. when all queries can be answered using the write batch only. + + ASSERT_OK(OpenDB()); + + constexpr size_t num_keys = 6; + + constexpr char delete_key[] = "d"; + constexpr char delete_merge_key[] = "dm"; + constexpr char put_entity_key[] = "e"; + constexpr char put_entity_merge_key[] = "em"; + constexpr char put_key[] = "p"; + constexpr char put_merge_key[] = "pm"; + + AddToBatch(db_->DefaultColumnFamily(), delete_key); + AddToBatch(db_->DefaultColumnFamily(), delete_merge_key); + AddToBatch(db_->DefaultColumnFamily(), put_entity_key); + AddToBatch(db_->DefaultColumnFamily(), put_entity_merge_key); + AddToBatch(db_->DefaultColumnFamily(), put_key); + AddToBatch(db_->DefaultColumnFamily(), put_merge_key); + + std::array keys{{delete_key, delete_merge_key, + put_entity_key, put_entity_merge_key, + put_key, put_merge_key}}; + + std::array expected{ + {{}, + {{kDefaultWideColumnName, "dm1"}}, + {{kDefaultWideColumnName, "e0"}, {"e", "0"}}, + {{kDefaultWideColumnName, "em0,em1"}, {"em", "0"}}, + {{kDefaultWideColumnName, "p0"}}, + {{kDefaultWideColumnName, "pm0,pm1"}}}}; + + // GetFromBatchAndDB + { + PinnableSlice value; + ASSERT_TRUE(batch_->GetFromBatchAndDB(db_, read_opts_, delete_key, &value) + .IsNotFound()); + } + + for (size_t i = 1; i < num_keys; ++i) { + PinnableSlice value; + ASSERT_OK(batch_->GetFromBatchAndDB(db_, read_opts_, keys[i], &value)); + ASSERT_EQ(value, expected[i].front().value()); + } + + // MultiGetFromBatchAndDB + { + std::array values; + std::array statuses; + constexpr bool sorted_input = false; + + batch_->MultiGetFromBatchAndDB(db_, read_opts_, db_->DefaultColumnFamily(), + num_keys, keys.data(), values.data(), + statuses.data(), sorted_input); + + ASSERT_TRUE(statuses[0].IsNotFound()); + + for (size_t i = 1; i < num_keys; ++i) { + ASSERT_OK(statuses[i]); + ASSERT_EQ(values[i], expected[i].front().value()); + } + } + + // TODO: add tests for GetEntityFromBatchAndDB and + // MultiGetEntityFromBatchAndDB once they are implemented + + // Iterator + std::unique_ptr iter(batch_->NewIteratorWithBase( + db_->DefaultColumnFamily(), db_->NewIterator(read_opts_), &read_opts_)); + + iter->SeekToFirst(); + + for (size_t i = 1; i < num_keys; ++i) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), keys[i]); + ASSERT_EQ(iter->value(), expected[i].front().value()); + ASSERT_EQ(iter->columns(), expected[i]); + iter->Next(); + } + + ASSERT_FALSE(iter->Valid()); + + iter->SeekToLast(); + + for (size_t i = num_keys - 1; i > 0; --i) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), keys[i]); + ASSERT_EQ(iter->value(), expected[i].front().value()); + ASSERT_EQ(iter->columns(), expected[i]); + iter->Prev(); + } + + ASSERT_FALSE(iter->Valid()); +} + +TEST_P(WriteBatchWithIndexTest, WideColumnsBatchAndDB) { + // Tests for the case when queries require consulting both the write batch and + // the underlying DB, either because of merges or because the write batch + // doesn't contain the key. + + ASSERT_OK(OpenDB()); + + constexpr size_t num_keys = 6; + + // Note: for the "merge" keys, we'll have a merge operation in the write batch + // and the base value (Put/PutEntity/Delete) in the DB. For the "no-merge" + // keys, we'll have nothing in the write batch and a standalone + // Put/PutEntity/Delete in the DB. + constexpr char merge_a_key[] = "ma"; + constexpr char merge_b_key[] = "mb"; + constexpr char merge_c_key[] = "mc"; + constexpr char no_merge_a_key[] = "na"; + constexpr char no_merge_b_key[] = "nb"; + constexpr char no_merge_c_key[] = "nc"; + + constexpr char merge_a_value[] = "mao"; + const WideColumns merge_b_columns{{kDefaultWideColumnName, "mbo"}, + {"mb", "o"}}; + constexpr char no_merge_a_value[] = "nao"; + const WideColumns no_merge_b_columns{{kDefaultWideColumnName, "nbo"}, + {"nb", "o"}}; + + ASSERT_OK(db_->Put(write_opts_, db_->DefaultColumnFamily(), merge_a_key, + merge_a_value)); + ASSERT_OK(db_->PutEntity(write_opts_, db_->DefaultColumnFamily(), merge_b_key, + merge_b_columns)); + ASSERT_OK(db_->Delete(write_opts_, db_->DefaultColumnFamily(), merge_c_key)); + ASSERT_OK(db_->Put(write_opts_, db_->DefaultColumnFamily(), no_merge_a_key, + no_merge_a_value)); + ASSERT_OK(db_->PutEntity(write_opts_, db_->DefaultColumnFamily(), + no_merge_b_key, no_merge_b_columns)); + ASSERT_OK( + db_->Delete(write_opts_, db_->DefaultColumnFamily(), no_merge_c_key)); + + AddToBatch(db_->DefaultColumnFamily(), merge_a_key); + AddToBatch(db_->DefaultColumnFamily(), merge_b_key); + AddToBatch(db_->DefaultColumnFamily(), merge_c_key); + + std::array keys{{merge_a_key, merge_b_key, merge_c_key, + no_merge_a_key, no_merge_b_key, + no_merge_c_key}}; + + std::array expected{ + {{{kDefaultWideColumnName, "mao,ma0"}}, + {{kDefaultWideColumnName, "mbo,mb0"}, {"mb", "o"}}, + {{kDefaultWideColumnName, "mc0"}}, + {{kDefaultWideColumnName, "nao"}}, + {{kDefaultWideColumnName, "nbo"}, {"nb", "o"}}, + {}}}; + + // GetFromBatchAndDB + for (size_t i = 0; i < num_keys - 1; ++i) { + PinnableSlice value; + ASSERT_OK(batch_->GetFromBatchAndDB(db_, read_opts_, keys[i], &value)); + ASSERT_EQ(value, expected[i].front().value()); + } + + { + PinnableSlice value; + ASSERT_TRUE( + batch_->GetFromBatchAndDB(db_, read_opts_, no_merge_c_key, &value) + .IsNotFound()); + } + + // MultiGetFromBatchAndDB + { + std::array values; + std::array statuses; + constexpr bool sorted_input = false; + + batch_->MultiGetFromBatchAndDB(db_, read_opts_, db_->DefaultColumnFamily(), + num_keys, keys.data(), values.data(), + statuses.data(), sorted_input); + + for (size_t i = 0; i < num_keys - 1; ++i) { + ASSERT_OK(statuses[i]); + ASSERT_EQ(values[i], expected[i].front().value()); + } + + ASSERT_TRUE(statuses[num_keys - 1].IsNotFound()); + } + + // TODO: add tests for GetEntityFromBatchAndDB and + // MultiGetEntityFromBatchAndDB once they are implemented + + // Iterator + std::unique_ptr iter(batch_->NewIteratorWithBase( + db_->DefaultColumnFamily(), db_->NewIterator(read_opts_), &read_opts_)); + + iter->SeekToFirst(); + + for (size_t i = 0; i < num_keys - 1; ++i) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), keys[i]); + ASSERT_EQ(iter->value(), expected[i].front().value()); + ASSERT_EQ(iter->columns(), expected[i]); + iter->Next(); + } + + ASSERT_FALSE(iter->Valid()); + + iter->SeekToLast(); + + for (size_t i = 0; i < num_keys - 1; ++i) { + const size_t idx = num_keys - 2 - i; + + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), keys[idx]); + ASSERT_EQ(iter->value(), expected[idx].front().value()); + ASSERT_EQ(iter->columns(), expected[idx]); + iter->Prev(); + } + + ASSERT_FALSE(iter->Valid()); +} + INSTANTIATE_TEST_CASE_P(WBWI, WriteBatchWithIndexTest, testing::Bool()); } // namespace ROCKSDB_NAMESPACE @@ -2406,4 +2774,3 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - diff --git a/zstd-1.5.5/tests/golden-decompression/empty-block.zst b/zstd-1.5.5/tests/golden-decompression/empty-block.zst new file mode 100644 index 000000000..fbfb893e1 Binary files /dev/null and b/zstd-1.5.5/tests/golden-decompression/empty-block.zst differ diff --git a/zstd-1.5.5/tests/golden-decompression/rle-first-block.zst b/zstd-1.5.5/tests/golden-decompression/rle-first-block.zst new file mode 100644 index 000000000..fd067edd7 Binary files /dev/null and b/zstd-1.5.5/tests/golden-decompression/rle-first-block.zst differ