Merge pull request #9 from anyproto/go-4369-research-jieba-add-weights

Get rid of base64
anyproto · Nov 5, 2024 · 51a7d0a · 51a7d0a
2 parents 246130d + fde73bf
commit 51a7d0a
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 6 deletions.
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -14,7 +14,6 @@ cbindgen = "0.27.0"
 tantivy = "0.22.0"
 cbindgen = "0.27.0"
 log = "0.4.21"
-base64 = "0.22.1"
 env_logger = "0.11.3"
 serde_json = "1.0.117"
 serde = { version = "1.0.203", features = ["derive"] }

diff --git a/rust/src/tantivy_util/highlights.rs b/rust/src/tantivy_util/highlights.rs
@@ -1,5 +1,4 @@
-use base64::Engine;
-use base64::engine::general_purpose;
+use log::debug;
 use tantivy::query::Query;
 use tantivy::{Searcher, SnippetGenerator, TantivyDocument, TantivyError};
 use tantivy::schema::Schema;
@@ -24,14 +23,13 @@ pub fn find_highlights(
                 .filter_map(|highlight| {
                     if highlight.is_empty() { None } else { Some((highlight.start, highlight.end)) }
                 }).collect();
-
             if highlighted.is_empty() {
                 continue;
             }
             highlights.push(Highlight {
                 field_name: schema.get_field_name(field_value.field).to_string(),
                 fragment: Fragment {
-                    t: general_purpose::STANDARD.encode(&snippet.fragment().to_owned()), //to comply with bleve temporarily
+                    t: snippet.fragment().to_owned(),
                     r: highlighted,
                 },
             });

diff --git a/tantivy_test.go b/tantivy_test.go
@@ -437,7 +437,7 @@ func Test(t *testing.T) {
 		sCtx := tantivy_go.NewSearchContextBuilder().
 			SetQuery("apple").
 			SetDocsLimit(100).
-			SetWithHighlights(false).
+			SetWithHighlights(true).
 			AddField(NameTitle, 1.0).
 			AddField(NameBody, 1.0).
 			Build()