From f3a36f8a299bdbb97238e67be4ff600344e87807 Mon Sep 17 00:00:00 2001 From: Zhen Ye Date: Wed, 20 Nov 2024 20:44:32 +0800 Subject: [PATCH] fix: use global pool but not dedicated pool for every index (#37852) issue: #37851 - make a global thread pool at tantivy temporally. - set 1 but not 4 threads for inverted text index. Signed-off-by: chyezh --- .../tantivy/tantivy-binding/Cargo.lock | 27 +++++++------------ .../tantivy/tantivy-binding/Cargo.toml | 2 +- .../core/thirdparty/tantivy/tantivy-wrapper.h | 3 ++- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock b/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock index 0232ecabe64f6..e4aefc8ad14c3 100644 --- a/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock +++ b/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock @@ -781,8 +781,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "ownedbytes" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "stable_deref_trait", ] @@ -1107,8 +1106,7 @@ dependencies = [ [[package]] name = "tantivy" version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6083cd777fa94271b8ce0fe4533772cb8110c3044bab048d20f70108329a1f2" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "aho-corasick", "arc-swap", @@ -1177,8 +1175,7 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "bitpacking", ] @@ -1186,8 +1183,7 @@ dependencies = [ [[package]] name = "tantivy-columnar" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "fastdivide", "fnv", @@ -1202,8 +1198,7 @@ dependencies = [ [[package]] name = "tantivy-common" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "async-trait", "byteorder", @@ -1226,8 +1221,7 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "nom", ] @@ -1235,8 +1229,7 @@ dependencies = [ [[package]] name = "tantivy-sstable" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "tantivy-common", "tantivy-fst", @@ -1246,8 +1239,7 @@ dependencies = [ [[package]] name = "tantivy-stacker" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "murmurhash32", "tantivy-common", @@ -1256,8 +1248,7 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66" +source = "git+https://github.com/milvus-io/tantivy?tag=0.21.1-fix#e2de49f01bbb9b9ba946a08ae05a8f495a691fbb" dependencies = [ "serde", ] diff --git a/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml b/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml index c0c857b168fbf..01473a0befef2 100644 --- a/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml +++ b/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tantivy = "=0.21.1" +tantivy = { git = "https://github.com/milvus-io/tantivy", tag = "0.21.1-fix" } # we have make a private fix for milvus, should be removed in future after milvus fixing the bug. futures = "0.3.21" libc = "0.2" scopeguard = "1.2" diff --git a/internal/core/thirdparty/tantivy/tantivy-wrapper.h b/internal/core/thirdparty/tantivy/tantivy-wrapper.h index d6e1cf3227320..09edc0d92337f 100644 --- a/internal/core/thirdparty/tantivy/tantivy-wrapper.h +++ b/internal/core/thirdparty/tantivy/tantivy-wrapper.h @@ -15,7 +15,8 @@ using Map = std::map; static constexpr const char* DEFAULT_TOKENIZER_NAME = "milvus_tokenizer"; static const char* DEFAULT_analyzer_params = "{}"; -static constexpr uintptr_t DEFAULT_NUM_THREADS = 4; +static constexpr uintptr_t DEFAULT_NUM_THREADS = + 1; // Every field with index writer will generate a thread, make huge thread amount, wait for refactoring. static constexpr uintptr_t DEFAULT_OVERALL_MEMORY_BUDGET_IN_BYTES = DEFAULT_NUM_THREADS * 15 * 1024 * 1024;