Skip to content

Commit

Permalink
chore(crawler): add default js enable intercept
Browse files Browse the repository at this point in the history
  • Loading branch information
j-mendez committed Nov 30, 2024
1 parent 21abbbd commit 499b2ed
Show file tree
Hide file tree
Showing 8 changed files with 15 additions and 17 deletions.
12 changes: 6 additions & 6 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion spider/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider"
version = "2.13.78"
version = "2.13.79"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
8 changes: 3 additions & 5 deletions spider/src/features/chrome_common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -613,11 +613,9 @@ impl WebAutomation {
pub fn set_dynamic_scroll(timeout: u32) -> String {
let timeout = timeout.min(60000);
let s = string_concat!(
r###"
document.addEventListener('DOMContentLoaded',e=>{let t=null,o=null,n="###,
r###"document.addEventListener('DOMContentLoaded',e=>{let t=null,o=null,n="###,
timeout.to_string(),
r###",a=Date.now(),i=Date.now(),r=()=>{window.scrollTo(0,document.body.scrollHeight)},l=()=>{o&&o.disconnect(),console.log('Stopped checking for new content.')},c=(e,n)=>{e.forEach(e=>{if(e.isIntersecting){i=Date.now();const n=Date.now();if(n-a>=t||n-i>=1e4)return void l();r(),t=document.querySelector('body > *:last-child'),o.observe(t)}})},s=()=>{t&&(o=new IntersectionObserver(c),o.observe(t))},d=()=>{['load','error','abort'].forEach(e=>{window.addEventListener(e,()=>{const e=document.querySelector('body > *:last-child');e!==t&&(i=Date.now(),t=e,o.observe(t))})})},u=()=>{r(),t=document.querySelector('body > *:last-child'),s(),d()};u(),setTimeout(l,n)});
"###
r###",a=Date.now(),i=Date.now(),r=()=>{window.scrollTo(0,document.body.scrollHeight)},l=()=>{o&&o.disconnect(),console.log('Stopped checking for new content.')},c=(e,n)=>{e.forEach(e=>{if(e.isIntersecting){i=Date.now();const n=Date.now();if(n-a>=t||n-i>=1e4)return void l();r(),t=document.querySelector('body > *:last-child'),o.observe(t)}})},s=()=>{t&&(o=new IntersectionObserver(c),o.observe(t))},d=()=>{['load','error','abort'].forEach(e=>{window.addEventListener(e,()=>{const e=document.querySelector('body > *:last-child');e!==t&&(i=Date.now(),t=e,o.observe(t))})})},u=()=>{r(),t=document.querySelector('body > *:last-child'),s(),d()};u(),setTimeout(l,n)});"###
);

s
Expand Down Expand Up @@ -654,8 +652,8 @@ impl RequestInterceptConfiguration {
pub fn new(enabled: bool) -> RequestInterceptConfiguration {
RequestInterceptConfiguration {
enabled,
block_javascript: false,
block_visuals: true,
block_javascript: true,
block_stylesheets: true,
..Default::default()
}
Expand Down
2 changes: 1 addition & 1 deletion spider_chrome/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_chrome"
version = "2.13.78"
version = "2.13.79"
rust-version = "1.70"
authors = [
"j-mendez <[email protected]>"
Expand Down
2 changes: 1 addition & 1 deletion spider_cli/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_cli"
version = "2.13.78"
version = "2.13.79"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
2 changes: 1 addition & 1 deletion spider_transformations/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_transformations"
version = "2.13.78"
version = "2.13.79"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
2 changes: 1 addition & 1 deletion spider_utils/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_utils"
version = "2.13.78"
version = "2.13.79"
authors = [
"j-mendez <[email protected]>"
]
Expand Down
2 changes: 1 addition & 1 deletion spider_worker/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "spider_worker"
version = "2.13.78"
version = "2.13.79"
authors = [
"j-mendez <[email protected]>"
]
Expand Down

0 comments on commit 499b2ed

Please sign in to comment.