feat(chrome): add chrome user agents list

spider-rs · Jun 13, 2024 · 6b217ea · 6b217ea
1 parent 2ae6139
commit 6b217ea
Show file tree

Hide file tree

Showing 8 changed files with 65 additions and 5 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/benches/Cargo.toml b/benches/Cargo.toml
@@ -5,7 +5,7 @@ publish = false
 edition = "2021"
 
 [dependencies]
-ua_generator = { version = "^0.4.0", path = "../ua_generator" }
+ua_generator = { version = "^0.5.0", path = "../ua_generator" }
 criterion = { version = "0.3.5", features = ["html_reports"] }
 
 [[bench]]

diff --git a/ua_generator/Cargo.toml b/ua_generator/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "ua_generator"
-version = "0.4.2"
+version = "0.5.0"
 edition = "2018"
 authors = ["j-mendez <[email protected]>"]
 description = "Random User Agent Spoofer in Rust."

diff --git a/ua_generator/build.rs b/ua_generator/build.rs
@@ -109,6 +109,34 @@ pub fn agents() -> [&'static str; 9] {{
         );
 
         fs::write(&dest_path, agents).unwrap();
+
+        // Build a list of valid chrome user-agents to use for chrome only browsers. 
+        let chrome_agent_list: Vec<ApiResult> =
+            match get(&format!("{base_api}?chrome=true&list=true"))
+                .set("apikey", &token)
+                .call()
+            {
+                Ok(req) => {
+                    let req: Vec<ApiResult> = req
+                        .into_json()
+                        .expect("Authorization not granted! Make sure to set a valid API key.");
+                    req
+                }
+                Err(e) => {
+                    panic!("{:?}", e)
+                }
+            };
+        let dest_path = Path::new(&"./src").join("chrome_ua_list.rs");
+        let mut chrome_devices = format!("/// List of real Chrome User-Agents.\n Duplicates may be stored to increase chances of getting the most relevant agent.\npub const STATIC_CHROME_AGENTS: &'static [&'static str; {}] = &[\n", chrome_agent_list.len());
+
+        for device in chrome_agent_list {
+            chrome_devices.push_str(&format!("    \"{}\",\n", device.agent));
+        }
+
+        chrome_devices.push_str("];");
+
+        fs::write(dest_path, chrome_devices).unwrap();
+
         println!("cargo:rerun-if-changed=build.rs");
     }
 

diff --git a/ua_generator/src/chrome_ua_list.rs b/ua_generator/src/chrome_ua_list.rs
@@ -0,0 +1,23 @@
+/// List of real Chrome User-Agents.
+pub const STATIC_CHROME_AGENTS: &'static [&'static str; 20] = &[
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+];
diff --git a/ua_generator/src/lib.rs b/ua_generator/src/lib.rs
@@ -20,5 +20,7 @@
 pub mod ua;
 /// Auto generated latest User Agents for windows, mac, linux, and android.
 pub mod ua_list;
+/// Auto generated list of real Chrome User Agents.
+pub mod chrome_ua_list;
 
 pub extern crate fastrand;
diff --git a/ua_generator/src/ua.rs b/ua_generator/src/ua.rs
@@ -1,4 +1,6 @@
 pub use crate::ua_list::STATIC_AGENTS;
+pub use crate::chrome_ua_list::STATIC_CHROME_AGENTS;
+
 use fastrand;
 use std::collections::HashMap;
 use std::rc::Rc;
@@ -8,6 +10,11 @@ pub fn spoof_ua() -> &'static str {
     STATIC_AGENTS[fastrand::usize(..STATIC_AGENTS.len())]
 }
 
+/// Get a random UA from a static precompiled list.
+pub fn spoof_chrome_ua() -> &'static str {
+    STATIC_CHROME_AGENTS[fastrand::usize(..STATIC_CHROME_AGENTS.len())]
+}
+
 /// Structure to manage a dynamic list of User-Agents, with quick lookup capabilities.
 #[derive(Default, Clone)]
 pub struct UserAgents {

diff --git a/ua_generator/src/ua_list.rs b/ua_generator/src/ua_list.rs
@@ -2,9 +2,9 @@
 pub const STATIC_AGENTS: &'static [&'static str; 9] = &[
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Android 12; Tablet; rv:110.0) Gecko/110.0 Firefox/110.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Android 12; Tablet; rv:110.0) Gecko/110.0 Firefox/110.0",
     "Mozilla/5.0 (X11; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",