chore(balance): add simple rebalance

spider/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.21.7"
+version = "2.21.8"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider/src/utils/detect_cpu.rs

@@ -1,20 +1,21 @@
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicI8, Ordering};
 use sysinfo::System;
 use tokio::sync::OnceCell;
 use tokio::time::sleep;
 
-/// Atomic value to store CPU usage.
-static CPU_USAGE: AtomicUsize = AtomicUsize::new(0);
+/// The CPU state for the crawl.
+///
+static CPU_STATE: AtomicI8 = AtomicI8::new(0);
 
 /// `OnceCell` CPU tracking.
 static INIT: OnceCell<()> = OnceCell::const_new();
 
 /// Get the total avg CPU being used.
-fn get_cpu_usage(sys: &System) -> usize {
+fn get_cpu_usage(sys: &System) -> f32 {
     sys.cpus()
         .iter()
         .map(|cpu| cpu.cpu_usage() / sys.cpus().len() as f32)
-        .sum::<f32>() as usize
+        .sum::<f32>()
 }
 
 /// Update the cpu usage being used.
@@ -25,7 +26,14 @@ async fn update_cpu_usage() {
         loop {
             sys.refresh_cpu_usage();
             let usage = get_cpu_usage(&sys);
-            CPU_USAGE.store(usage, Ordering::Relaxed);
+            let state = if usage >= 70.0 {
+                1
+            } else if usage >= 95.0 {
+                2
+            } else {
+                0
+            };
+            CPU_STATE.store(state, Ordering::Relaxed);
             sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL).await;
         }
     }
@@ -40,7 +48,7 @@ async fn init_once() {
 }
 
 /// Get the cpu usage being used utility.
-pub async fn get_global_cpu_usage() -> usize {
+pub async fn get_global_cpu_usage() -> i8 {
     init_once().await;
-    CPU_USAGE.load(Ordering::Relaxed)
+    CPU_STATE.load(Ordering::Relaxed)
 }

spider/src/utils/mod.rs

@@ -2998,6 +2998,10 @@ where
     set.spawn(future)
 }
 
+#[cfg(feature = "balance")]
+/// Period to wait to rebalance cpu in means of IO being main impact.
+const REBALANCE_TIME: std::time::Duration = std::time::Duration::from_millis(100);
+
 /// Return the semaphore that should be used.
 #[cfg(feature = "balance")]
 pub async fn get_semaphore(semaphore: &Arc<Semaphore>, detect: bool) -> &Arc<Semaphore> {
@@ -3007,7 +3011,11 @@ pub async fn get_semaphore(semaphore: &Arc<Semaphore>, detect: bool) -> &Arc<Sem
         0
     };
 
-    if cpu_load >= 70 {
+    if cpu_load == 2 {
+        tokio::time::sleep(REBALANCE_TIME).await;
+    }
+
+    if cpu_load >= 1 {
         &*crate::website::SEM_SHARED
     } else {
         semaphore

spider_chrome/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.21.7"
+version = "2.21.8"
 rust-version = "1.70"
 authors = [
     "j-mendez <[email protected]>"

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.21.7"
+version = "2.21.8"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_transformations/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.21.7"
+version = "2.21.8"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.21.7"
+version = "2.21.8"
 authors = [
     "j-mendez <[email protected]>"
 ]

spider_worker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.21.7"
+version = "2.21.8"
 authors = [
     "j-mendez <[email protected]>"
 ]