chore(transform): fix streaming repeated text

spider-rs · Dec 3, 2024 · a405f82 · a405f82
1 parent 8d540e8
commit a405f82
Show file tree

Hide file tree

Showing 8 changed files with 27 additions and 16 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.13.96"
+version = "2.13.97"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.13.96"
+version = "2.13.97"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.13.96"
+version = "2.13.97"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.13.96"
+version = "2.13.97"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/src/transformation/text_extract.rs b/spider_transformations/src/transformation/text_extract.rs
@@ -87,7 +87,7 @@ pub async fn extract_text_streaming_with_size(
     }
 
     let mut extracted_text = String::new();
-    let txx1 = txx.clone();
+    let mut last_sent_position = 0;
 
     element_content_handlers.push(text!(
         "*:not(script):not(style):not(svg):not(noscript):not(nav):not(footer)",
@@ -105,7 +105,20 @@ pub async fn extract_text_streaming_with_size(
                     extracted_text.push('\n');
                 }
 
-                let _ = txx1.send(extracted_text.clone());
+                let new_slice = &extracted_text[last_sent_position..];
+
+                if !new_slice.is_empty() {
+                    let _ = txx.send(new_slice.to_string());
+                    last_sent_position = extracted_text.len();
+                }
+
+                // clear the text tracker
+                if extracted_text.len() > 1024 {
+                    if !extracted_text.ends_with(' ') {
+                        extracted_text.clear();
+                        last_sent_position = 0;
+                    }
+                }
             }
 
             Ok(())
@@ -138,8 +151,6 @@ pub async fn extract_text_streaming_with_size(
         let _ = rewriter.end();
     }
 
-    drop(txx);
-
     let mut rewrited_bytes: String = String::new();
 
     while let Some(c) = rxx.recv().await {

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.13.96"
+version = "2.13.97"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.13.96"
+version = "2.13.97"
 authors = [
     "j-mendez <[email protected]>"
 ]