diff --git a/Cargo.lock b/Cargo.lock index 0c310f86e..6a08cda67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4273,7 +4273,7 @@ dependencies = [ [[package]] name = "spider" -version = "2.21.17" +version = "2.21.18" dependencies = [ "ahash", "aho-corasick", @@ -4336,7 +4336,7 @@ dependencies = [ [[package]] name = "spider_chrome" -version = "2.21.17" +version = "2.21.18" dependencies = [ "adblock", "base64 0.22.1", @@ -4373,7 +4373,7 @@ dependencies = [ [[package]] name = "spider_cli" -version = "2.21.17" +version = "2.21.18" dependencies = [ "clap", "env_logger", @@ -4398,7 +4398,7 @@ dependencies = [ [[package]] name = "spider_transformations" -version = "2.21.17" +version = "2.21.18" dependencies = [ "aho-corasick", "fast_html2md", @@ -4420,7 +4420,7 @@ dependencies = [ [[package]] name = "spider_utils" -version = "2.21.17" +version = "2.21.18" dependencies = [ "indexmap 1.9.3", "serde", @@ -4432,7 +4432,7 @@ dependencies = [ [[package]] name = "spider_worker" -version = "2.21.17" +version = "2.21.18" dependencies = [ "env_logger", "lazy_static", diff --git a/spider/Cargo.toml b/spider/Cargo.toml index 087e50fa2..aedbc77ec 100644 --- a/spider/Cargo.toml +++ b/spider/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider" -version = "2.21.17" +version = "2.21.18" authors = [ "j-mendez " ] diff --git a/spider/src/configuration.rs b/spider/src/configuration.rs index ff90236e3..2415691c0 100644 --- a/spider/src/configuration.rs +++ b/spider/src/configuration.rs @@ -643,7 +643,7 @@ impl Configuration { self } - /// Set a crawl page limit. If the value is 0 there is no limit. This does nothing without the feat flag `budget` enabled. + /// Set a crawl page limit. If the value is 0 there is no limit. pub fn with_limit(&mut self, limit: u32) -> &mut Self { self.with_budget(Some(hashbrown::HashMap::from([("*", limit)]))); self @@ -693,7 +693,7 @@ impl Configuration { self } - /// Set a crawl depth limit. If the value is 0 there is no limit. This does nothing without the feat flag `budget` enabled. + /// Set a crawl depth limit. If the value is 0 there is no limit. pub fn with_depth(&mut self, depth: usize) -> &mut Self { self.depth = depth; self diff --git a/spider/src/website.rs b/spider/src/website.rs index 7a19898e5..fd0240292 100644 --- a/spider/src/website.rs +++ b/spider/src/website.rs @@ -3914,7 +3914,7 @@ impl Website { self.configuration.budget = budget; } - /// Set a crawl depth limit. If the value is 0 there is no limit. This does nothing without the feat flag `budget` enabled. + /// Set a crawl depth limit. If the value is 0 there is no limit. pub fn with_depth(&mut self, depth: usize) -> &mut Self { self.configuration.with_depth(depth); self @@ -4079,7 +4079,7 @@ impl Website { self } - /// Set a crawl page limit. If the value is 0 there is no limit. This does nothing without the feat flag `budget` enabled. + /// Set a crawl page limit. If the value is 0 there is no limit. pub fn with_limit(&mut self, limit: u32) -> &mut Self { self.configuration.with_limit(limit); self diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml index ead6166eb..653587dfc 100644 --- a/spider_chrome/Cargo.toml +++ b/spider_chrome/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_chrome" -version = "2.21.17" +version = "2.21.18" rust-version = "1.70" authors = [ "j-mendez " diff --git a/spider_chrome/src/handler/blockers/intercept_manager.rs b/spider_chrome/src/handler/blockers/intercept_manager.rs index e70891074..341f84568 100644 --- a/spider_chrome/src/handler/blockers/intercept_manager.rs +++ b/spider_chrome/src/handler/blockers/intercept_manager.rs @@ -77,7 +77,6 @@ impl NetworkInterceptManager { } } - #[cfg(test)] mod tests { use super::*; @@ -95,15 +94,30 @@ mod tests { ("https://facebook.com", NetworkInterceptManager::Facebook), ("https://www.amazon.com", NetworkInterceptManager::Amazon), ("https://subdomain.x.com", NetworkInterceptManager::X), - ("https://linkedin.com/in/someone", NetworkInterceptManager::LinkedIn), - ("https://www.netflix.com/browse", NetworkInterceptManager::Netflix), + ( + "https://linkedin.com/in/someone", + NetworkInterceptManager::LinkedIn, + ), + ( + "https://www.netflix.com/browse", + NetworkInterceptManager::Netflix, + ), ("https://medium.com", NetworkInterceptManager::Medium), ("https://sub.upwork.com", NetworkInterceptManager::Upwork), ("https://glassdoor.com", NetworkInterceptManager::Glassdoor), ("https://ebay.com", NetworkInterceptManager::Ebay), - ("https://nytimes.com/section/world", NetworkInterceptManager::Nytimes), - ("https://en.wikipedia.org/wiki/Rust", NetworkInterceptManager::Wikipedia), - ("https://market.tcgplayer.com", NetworkInterceptManager::Tcgplayer), + ( + "https://nytimes.com/section/world", + NetworkInterceptManager::Nytimes, + ), + ( + "https://en.wikipedia.org/wiki/Rust", + NetworkInterceptManager::Wikipedia, + ), + ( + "https://market.tcgplayer.com", + NetworkInterceptManager::Tcgplayer, + ), ]; for (url, expected) in cases { @@ -121,21 +135,22 @@ mod tests { ]; for url in cases { - assert_eq!(NetworkInterceptManager::new(&create_url(url)), NetworkInterceptManager::Unknown); + assert_eq!( + NetworkInterceptManager::new(&create_url(url)), + NetworkInterceptManager::Unknown + ); } } #[test] fn test_invalid_urls() { - let cases = vec![ - "not-a-url", - "ftp://invalid.protocol.com", - "http://", - "", - ]; + let cases = vec!["not-a-url", "ftp://invalid.protocol.com", "http://", ""]; for url in cases { - assert_eq!(NetworkInterceptManager::new(&create_url(url)), NetworkInterceptManager::Unknown); + assert_eq!( + NetworkInterceptManager::new(&create_url(url)), + NetworkInterceptManager::Unknown + ); } } -} \ No newline at end of file +} diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml index f08e892e6..f85e99db3 100644 --- a/spider_cli/Cargo.toml +++ b/spider_cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_cli" -version = "2.21.17" +version = "2.21.18" authors = [ "j-mendez " ] diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml index 86c0e5442..c7f8207b6 100644 --- a/spider_transformations/Cargo.toml +++ b/spider_transformations/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_transformations" -version = "2.21.17" +version = "2.21.18" authors = [ "j-mendez " ] diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml index e4a51132c..47f6ce51d 100644 --- a/spider_utils/Cargo.toml +++ b/spider_utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_utils" -version = "2.21.17" +version = "2.21.18" authors = [ "j-mendez " ] diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml index 4d9e993d2..b7af2ade9 100644 --- a/spider_worker/Cargo.toml +++ b/spider_worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider_worker" -version = "2.21.17" +version = "2.21.18" authors = [ "j-mendez " ]