chore(chrome): fix allowed critical rendering script interception

spider-rs · Dec 14, 2024 · 36c15d1 · 36c15d1
1 parent 3cfacdb
commit 36c15d1
Show file tree

Hide file tree

Showing 10 changed files with 50 additions and 34 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/examples/real_world.rs b/examples/real_world.rs
@@ -1,4 +1,4 @@
-//! cargo run --example real_world --features="chrome chrome_intercept real_browser spider_utils/transformations"
+//! cargo run --example real_world --features="chrome chrome_intercept real_browser"
 
 extern crate spider;
 use crate::spider::tokio::io::AsyncWriteExt;
@@ -13,10 +13,13 @@ use std::time::Duration;
 
 async fn crawl_website(url: &str) -> Result<()> {
     let mut stdout = tokio::io::stdout();
+    let mut interception = RequestInterceptConfiguration::new(true);
+
+    interception.block_javascript = true;
 
     let mut website: Website = Website::new(url)
         .with_limit(5)
-        .with_chrome_intercept(RequestInterceptConfiguration::new(true))
+        .with_chrome_intercept(interception)
         .with_wait_for_idle_network(Some(WaitForIdleNetwork::new(Some(Duration::from_millis(
             500,
         )))))
@@ -82,7 +85,7 @@ async fn crawl_website(url: &str) -> Result<()> {
 async fn main() -> Result<()> {
     env_logger::init();
     let _ = tokio::join!(
-        crawl_website("https://www.choosealicense.com"),
+        crawl_website("https://choosealicense.com"),
         crawl_website("https://jeffmendez.com"),
         crawl_website("https://example.com"),
     );

diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.21.21"
+version = "2.21.23"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.21.21"
+version = "2.21.23"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"
@@ -53,6 +53,7 @@ adblock = { version = "0.9", optional = true, default-features = false, features
 rand = "0.8"
 case_insensitive_string = { version = "0.2", features = ["compact", "serde"] }
 hashbrown = { version = "0.15", default-features = true }
+aho-corasick = "1"
 
 [target.'cfg(windows)'.dependencies]
 winreg = "0.52"

diff --git a/spider_chrome/src/handler/blockers/xhr.rs b/spider_chrome/src/handler/blockers/xhr.rs
@@ -95,6 +95,7 @@ lazy_static::lazy_static! {
             // video embeddings
             "https://video.squarespace-cdn.com/content/",
             "https://bes.gcp.data.bigcommerce.com/nobot",
+            "https://www.youtube.com/youtubei/",
             "http://ec.editmysite.com",
             "https://dcinfos-cache.abtasty.com/",
             "https://featureassets.org/",

diff --git a/spider_chrome/src/handler/network.rs b/spider_chrome/src/handler/network.rs
@@ -10,6 +10,7 @@ use super::blockers::{
 use crate::auth::Credentials;
 use crate::cmd::CommandChain;
 use crate::handler::http::HttpRequest;
+use aho_corasick::AhoCorasick;
 use case_insensitive_string::CaseInsensitiveString;
 use chromiumoxide_cdp::cdp::browser_protocol::fetch::{
     self, AuthChallengeResponse, AuthChallengeResponseResponse, ContinueRequestParams,
@@ -32,23 +33,32 @@ use std::collections::VecDeque;
 use std::time::Duration;
 
 lazy_static! {
-    /// allowed js frameworks and libs excluding some and adding additional URLs
-    pub static ref JS_FRAMEWORK_ALLOW: phf::Set<&'static str> = {
-        phf::phf_set! {
-            // Add allowed assets from JS_FRAMEWORK_ASSETS except the excluded ones
-            "jquery.min.js", "jquery.qtip.min.js", "jquery.js", "angular.js", "jquery.slim.js",
-            "react.development.js", "react-dom.development.js", "react.production.min.js",
-            "react-dom.production.min.js", "vue.global.js", "vue.esm-browser.js", "vue.js",
-            "bootstrap.min.js", "bootstrap.bundle.min.js", "bootstrap.esm.min.js", "d3.min.js",
-            "d3.js", "lodash.min.js", "lodash.js",
-            "app.js", "main.js", "index.js", "bundle.js", "vendor.js",
-            // Verified 3rd parties for request
-            "https://m.stripe.network/inner.html",
-            "https://m.stripe.network/out-4.5.43.js",
-            "https://challenges.cloudflare.com/turnstile",
-            "https://js.stripe.com/v3/"
-        }
-    };
+    /// General patterns for popular libraries and resources
+    static ref JS_FRAMEWORK_ALLOW: Vec<&'static str> = vec![
+        "jquery",           // Covers jquery.min.js, jquery.js, etc.
+        "angular",
+        "react",            // Covers all React-related patterns
+        "vue",              // Covers all Vue-related patterns
+        "bootstrap",
+        "d3",
+        "lodash",
+        "ajax",
+        "app",              // Covers general app scripts like app.js
+        "main",
+        "index",
+        "bundle",
+        "vendor",
+        "/wp-content/js/",  // Covers word press content
+        // Verified 3rd parties for request
+        "https://m.stripe.network/",
+        "https://challenges.cloudflare.com/",
+        "https://js.stripe.com/",
+        "https://cdn.prod.website-files.com/", // webflow cdn scripts
+        "https://cdnjs.cloudflare.com/"        // cloudflare cdn scripts
+    ];
+
+    /// Create a static AhoCorasick matcher based on the allowed list
+    static ref ALLOWED_MATCHER: AhoCorasick = AhoCorasick::new(JS_FRAMEWORK_ALLOW.iter()).unwrap();
 
     /// path of a js framework
     pub static ref JS_FRAMEWORK_PATH: phf::Set<&'static str> = {
@@ -400,7 +410,7 @@ impl NetworkManager {
                                 && ResourceType::Stylesheet == event.resource_type
                             || self.block_javascript
                                 && javascript_resource
-                                && !JS_FRAMEWORK_ALLOW.contains(current_url)
+                                && !ALLOWED_MATCHER.is_match(current_url)
                     } else {
                         skip_networking
                     };
@@ -502,7 +512,7 @@ impl NetworkManager {
                                 && ResourceType::Stylesheet == event.resource_type
                             || self.block_javascript
                                 && javascript_resource
-                                && !JS_FRAMEWORK_ALLOW.contains(current_url)
+                                && !ALLOWED_MATCHER.contains(current_url)
                     } else {
                         skip_networking
                     };

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.21.21"
+version = "2.21.23"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.21.21"
+version = "2.21.23"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.21.21"
+version = "2.21.23"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.21.21"
+version = "2.21.23"
 authors = [
     "j-mendez <[email protected]>"
 ]