fix: use global pool but not dedicated pool for every index (#37852)

issue: #37851 - make a global thread pool at tantivy temporally. - set 1 but not 4 threads for inverted text index. Signed-off-by: chyezh <[email protected]>
milvus-io · Nov 20, 2024 · f3a36f8 · f3a36f8
1 parent 7144219
commit f3a36f8
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 20 deletions.
diff --git a/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock b/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock
diff --git a/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml b/internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tantivy = "=0.21.1"
+tantivy = { git = "https://github.com/milvus-io/tantivy", tag = "0.21.1-fix" } # we have make a private fix for milvus, should be removed in future after milvus fixing the bug.
 futures = "0.3.21"
 libc = "0.2"
 scopeguard = "1.2"

diff --git a/internal/core/thirdparty/tantivy/tantivy-wrapper.h b/internal/core/thirdparty/tantivy/tantivy-wrapper.h
@@ -15,7 +15,8 @@ using Map = std::map<std::string, std::string>;
 
 static constexpr const char* DEFAULT_TOKENIZER_NAME = "milvus_tokenizer";
 static const char* DEFAULT_analyzer_params = "{}";
-static constexpr uintptr_t DEFAULT_NUM_THREADS = 4;
+static constexpr uintptr_t DEFAULT_NUM_THREADS =
+    1;  // Every field with index writer will generate a thread, make huge thread amount, wait for refactoring.
 static constexpr uintptr_t DEFAULT_OVERALL_MEMORY_BUDGET_IN_BYTES =
     DEFAULT_NUM_THREADS * 15 * 1024 * 1024;