chore(chrome): fix remote page closing

examples/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_examples"
-version = "1.40.10"
+version = "1.40.11"
 authors = ["madeindjs <[email protected]>", "j-mendez <[email protected]>"]
 description = "Multithreaded web crawler written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -22,7 +22,7 @@ htr = "0.5.27"
 flexbuffers = "2.0.0"
 
 [dependencies.spider]
-version = "1.40.10"
+version = "1.40.11"
 path = "../spider"
 features = ["serde"]
 

spider/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "1.40.10"
+version = "1.40.11"
 authors = ["madeindjs <[email protected]>", "j-mendez <[email protected]>"]
 description = "The fastest web crawler written in Rust."
 repository = "https://github.com/spider-rs/spider"

spider/README.md

@@ -16,7 +16,7 @@ This is a basic async example crawling a web page, add spider to your `Cargo.tom
 
 ```toml
 [dependencies]
-spider = "1.40.10"
+spider = "1.40.11"
 ```
 
 And then the code:
@@ -87,7 +87,7 @@ We have a couple optional feature flags. Regex blacklisting, jemaloc backend, gl
 
 ```toml
 [dependencies]
-spider = { version = "1.40.10", features = ["regex", "ua_generator"] }
+spider = { version = "1.40.11", features = ["regex", "ua_generator"] }
 ```
 
 1. `ua_generator`: Enables auto generating a random real User-Agent.
@@ -114,7 +114,7 @@ Move processing to a worker, drastically increases performance even if worker is
 
 ```toml
 [dependencies]
-spider = { version = "1.40.10", features = ["decentralized"] }
+spider = { version = "1.40.11", features = ["decentralized"] }
 ```
 
 ```sh
@@ -135,7 +135,7 @@ Use the subscribe method to get a broadcast channel.
 
 ```toml
 [dependencies]
-spider = { version = "1.40.10", features = ["sync"] }
+spider = { version = "1.40.11", features = ["sync"] }
 ```
 
 ```rust,no_run
@@ -165,7 +165,7 @@ Allow regex for blacklisting routes
 
 ```toml
 [dependencies]
-spider = { version = "1.40.10", features = ["regex"] }
+spider = { version = "1.40.11", features = ["regex"] }
 ```
 
 ```rust,no_run
@@ -192,7 +192,7 @@ If you are performing large workloads you may need to control the crawler by ena
 
 ```toml
 [dependencies]
-spider = { version = "1.40.10", features = ["control"] }
+spider = { version = "1.40.11", features = ["control"] }
 ```
 
 ```rust

spider/src/features/chrome.rs

@@ -81,7 +81,8 @@ pub async fn launch_browser(
     let (browser, mut handler) = match std::env::var("CHROME_URL") {
         Ok(v) => Browser::connect(&v).await,
         _ => Browser::launch(get_browser_config(&proxies).unwrap()).await,
-    }.unwrap();
+    }
+    .unwrap();
 
     // spawn a new task that continuously polls the handler
     let handle = task::spawn(async move {

spider/src/website.rs

@@ -875,6 +875,7 @@ impl Website {
             let (mut browser, browser_handle) = launch_browser(&self.configuration.proxies).await;
 
             let new_page = browser.new_page("about:blank").await.unwrap();
+
             if cfg!(feature = "chrome_stealth") {
                 let _ = new_page.enable_stealth_mode(&if self.configuration.user_agent.is_some() {
                     &self.configuration.user_agent.as_ref().unwrap().as_str()
@@ -887,7 +888,7 @@ impl Website {
                 client,
                 unsafe { selectors.unwrap_unchecked() },
                 self.channel.clone(),
-                new_page,
+                new_page.clone(),
             ));
 
             let mut links: HashSet<CaseInsensitiveString> = self
@@ -983,6 +984,8 @@ impl Website {
             if !std::env::var("CHROME_URL").is_ok() {
                 let _ = browser.close().await;
                 let _ = browser_handle.await;
+            } else {
+                let _ = new_page.close().await;
             }
         }
     }
@@ -1342,8 +1345,7 @@ impl Website {
                     ""
                 });
             }
-            let page = Arc::new(new_page);
-
+            let page = Arc::new(new_page.clone());
             // crawl while links exists
             loop {
                 let stream =
@@ -1437,6 +1439,8 @@ impl Website {
 
             if !std::env::var("CHROME_URL").is_ok() {
                 let _ = browser.close().await;
+            } else {
+                let _ = new_page.close().await;
             }
         }
     }

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "1.40.10"
+version = "1.40.11"
 authors = ["madeindjs <[email protected]>", "j-mendez <[email protected]>"]
 description = "The fastest web crawler CLI written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -26,7 +26,7 @@ quote = "1.0.18"
 failure_derive = "0.1.8"
 
 [dependencies.spider]
-version = "1.40.10"
+version = "1.40.11"
 path = "../spider"
 
 [[bin]]

spider_cli/README.md

@@ -40,7 +40,7 @@ spider --domain http://localhost:3000 download -t _temp_spider_downloads
 ```
 
 ```sh
-spider_cli 1.40.10
+spider_cli 1.40.11
 madeindjs <[email protected]>, j-mendez <[email protected]>
 The fastest web crawler CLI written in Rust.
 

spider_worker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "1.40.10"
+version = "1.40.11"
 authors = ["madeindjs <[email protected]>", "j-mendez <[email protected]>"]
 description = "The fastest web crawler CLI written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -22,7 +22,7 @@ lazy_static = "1.4.0"
 env_logger = "0.10.0"
 
 [dependencies.spider]
-version = "1.40.10"
+version = "1.40.11"
 path = "../spider"
 features = ["serde", "flexbuffers"]