diff --git a/Cargo.lock b/Cargo.lock index c1b32d93b..92ea86f92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4304,7 +4304,7 @@ dependencies = [ [[package]] name = "spider" -version = "2.13.78" +version = "2.13.79" dependencies = [ "ahash", "aho-corasick", @@ -4366,7 +4366,7 @@ dependencies = [ [[package]] name = "spider_chrome" -version = "2.13.78" +version = "2.13.79" dependencies = [ "adblock", "async-tungstenite", @@ -4401,7 +4401,7 @@ dependencies = [ [[package]] name = "spider_cli" -version = "2.13.78" +version = "2.13.79" dependencies = [ "clap", "env_logger", @@ -4426,7 +4426,7 @@ dependencies = [ [[package]] name = "spider_transformations" -version = "2.13.78" +version = "2.13.79" dependencies = [ "aho-corasick", "fast_html2md", @@ -4448,7 +4448,7 @@ dependencies = [ [[package]] name = "spider_utils" -version = "2.13.78" +version = "2.13.79" dependencies = [ "indexmap 1.9.3", "serde", @@ -4460,7 +4460,7 @@ dependencies = [ [[package]] name = "spider_worker" -version = "2.13.78" +version = "2.13.79" dependencies = [ "env_logger", "lazy_static", diff --git a/spider/Cargo.toml b/spider/Cargo.toml index b66ef292e..523ef8dba 100644 --- a/spider/Cargo.toml +++ b/spider/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider" -version = "2.13.78" +version = "2.13.79" authors = [ "j-mendez " ] diff --git a/spider/src/features/chrome_common.rs b/spider/src/features/chrome_common.rs index 5a6a7793e..583e91660 100644 --- a/spider/src/features/chrome_common.rs +++ b/spider/src/features/chrome_common.rs @@ -613,11 +613,9 @@ impl WebAutomation { pub fn set_dynamic_scroll(timeout: u32) -> String { let timeout = timeout.min(60000); let s = string_concat!( - r###" - document.addEventListener('DOMContentLoaded',e=>{let t=null,o=null,n="###, + r###"document.addEventListener('DOMContentLoaded',e=>{let t=null,o=null,n="###, timeout.to_string(), - r###",a=Date.now(),i=Date.now(),r=()=>{window.scrollTo(0,document.body.scrollHeight)},l=()=>{o&&o.disconnect(),console.log('Stopped checking for new content.')},c=(e,n)=>{e.forEach(e=>{if(e.isIntersecting){i=Date.now();const n=Date.now();if(n-a>=t||n-i>=1e4)return void l();r(),t=document.querySelector('body > *:last-child'),o.observe(t)}})},s=()=>{t&&(o=new IntersectionObserver(c),o.observe(t))},d=()=>{['load','error','abort'].forEach(e=>{window.addEventListener(e,()=>{const e=document.querySelector('body > *:last-child');e!==t&&(i=Date.now(),t=e,o.observe(t))})})},u=()=>{r(),t=document.querySelector('body > *:last-child'),s(),d()};u(),setTimeout(l,n)}); - "### + r###",a=Date.now(),i=Date.now(),r=()=>{window.scrollTo(0,document.body.scrollHeight)},l=()=>{o&&o.disconnect(),console.log('Stopped checking for new content.')},c=(e,n)=>{e.forEach(e=>{if(e.isIntersecting){i=Date.now();const n=Date.now();if(n-a>=t||n-i>=1e4)return void l();r(),t=document.querySelector('body > *:last-child'),o.observe(t)}})},s=()=>{t&&(o=new IntersectionObserver(c),o.observe(t))},d=()=>{['load','error','abort'].forEach(e=>{window.addEventListener(e,()=>{const e=document.querySelector('body > *:last-child');e!==t&&(i=Date.now(),t=e,o.observe(t))})})},u=()=>{r(),t=document.querySelector('body > *:last-child'),s(),d()};u(),setTimeout(l,n)});"### ); s @@ -654,8 +652,8 @@ impl RequestInterceptConfiguration { pub fn new(enabled: bool) -> RequestInterceptConfiguration { RequestInterceptConfiguration { enabled, + block_javascript: false, block_visuals: true, - block_javascript: true, block_stylesheets: true, ..Default::default() } diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml index 11fe9da98..d78d3d04a 100644 --- a/spider_chrome/Cargo.toml +++ b/spider_chrome/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_chrome" -version = "2.13.78" +version = "2.13.79" rust-version = "1.70" authors = [ "j-mendez " diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml index 61dc1da57..be65b05d2 100644 --- a/spider_cli/Cargo.toml +++ b/spider_cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_cli" -version = "2.13.78" +version = "2.13.79" authors = [ "j-mendez " ] diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml index c7acee07b..f59d4b753 100644 --- a/spider_transformations/Cargo.toml +++ b/spider_transformations/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_transformations" -version = "2.13.78" +version = "2.13.79" authors = [ "j-mendez " ] diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml index 66132bcaf..43791efcc 100644 --- a/spider_utils/Cargo.toml +++ b/spider_utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_utils" -version = "2.13.78" +version = "2.13.79" authors = [ "j-mendez " ] diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml index 024e64b6b..2dc680d42 100644 --- a/spider_worker/Cargo.toml +++ b/spider_worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_worker" -version = "2.13.78" +version = "2.13.79" authors = [ "j-mendez " ]