diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..1fc078182 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] +rustdocflags = ["--cfg", "tokio_unstable"] diff --git a/Cargo.lock b/Cargo.lock index 2ca0f5233..ae8818eea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -204,7 +204,7 @@ dependencies = [ "polling", "rustix 0.37.3", "slab", - "socket2", + "socket2 0.4.9", "waker-fn", ] @@ -2209,7 +2209,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -3382,9 +3382,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -3810,9 +3810,9 @@ checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64 0.21.0", "bytes", @@ -3836,6 +3836,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-rustls 0.24.1", "tower-service", @@ -3843,7 +3844,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots 0.25.2", "winreg", ] @@ -3957,6 +3958,7 @@ dependencies = [ "serde_json", "sscanf", "tokio", + "tokio-metrics", "tokio-rustls 0.24.1", "tokio-util", "tracing", @@ -4177,6 +4179,7 @@ dependencies = [ "ethers", "futures", "rand", + "reqwest", "tokio", "tracing", "url", @@ -4778,6 +4781,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "soketto" version = "0.7.1" @@ -4986,6 +4999,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" @@ -5141,11 +5175,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.29.1" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ - "autocfg", "backtrace", "bytes", "libc", @@ -5154,7 +5187,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.4", "tokio-macros", "windows-sys 0.48.0", ] @@ -5180,6 +5213,18 @@ dependencies = [ "syn 2.0.32", ] +[[package]] +name = "tokio-metrics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eace09241d62c98b7eeb1107d4c5c64ca3bd7da92e8c218c153ab3a78f9be112" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-rustls" version = "0.23.4" @@ -5759,15 +5804,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "webpki-roots" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki", -] - [[package]] name = "webpki-roots" version = "0.23.0" @@ -6002,11 +6038,12 @@ dependencies = [ [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys 0.48.0", ] [[package]] diff --git a/bin/rundler/Cargo.toml b/bin/rundler/Cargo.toml index 7f807cb40..c1c410c6b 100644 --- a/bin/rundler/Cargo.toml +++ b/bin/rundler/Cargo.toml @@ -35,6 +35,7 @@ serde.workspace = true serde_json.workspace = true sscanf = "0.4.0" tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal", "sync"] } +tokio-metrics = "0.3.1" tokio-rustls = "0.24.1" tokio-util = "0.7.8" tracing.workspace = true diff --git a/bin/rundler/src/cli/metrics.rs b/bin/rundler/src/cli/metrics.rs new file mode 100644 index 000000000..1b5c04866 --- /dev/null +++ b/bin/rundler/src/cli/metrics.rs @@ -0,0 +1,227 @@ +// This file is part of Rundler. +// +// Rundler is free software: you can redistribute it and/or modify it under the +// terms of the GNU Lesser General Public License as published by the Free Software +// Foundation, either version 3 of the License, or (at your option) any later version. +// +// Rundler is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along with Rundler. +// If not, see https://www.gnu.org/licenses/. + +use std::{net::SocketAddr, time::Duration}; + +use itertools::Itertools; +use metrics::gauge; +use metrics_exporter_prometheus::PrometheusBuilder; +use metrics_process::Collector; +use metrics_util::layers::{PrefixLayer, Stack}; + +pub fn initialize<'a>( + sample_interval_millis: u64, + listen_addr: SocketAddr, + tags: impl IntoIterator, +) -> anyhow::Result<()> { + let mut builder = PrometheusBuilder::new().with_http_listener(listen_addr); + + let tags: Vec<(&str, &str)> = tags + .into_iter() + .filter_map(|t| t.split('=').collect_tuple()) + .collect(); + for (k, v) in tags { + builder = builder.add_global_label(k, v); + } + + let (recorder, exporter) = builder.build()?; + tokio::spawn(exporter); + Stack::new(recorder) + .push(PrefixLayer::new("rundler")) + .install()?; + + tokio::spawn(async move { + let collector = Collector::default(); + loop { + collector.collect(); + tokio::time::sleep(Duration::from_millis(sample_interval_millis)).await; + } + }); + + let handle = tokio::runtime::Handle::current(); + let frequency = std::time::Duration::from_millis(sample_interval_millis); + let runtime_metrics = handle.metrics(); + let runtime_monitor = tokio_metrics::RuntimeMonitor::new(&handle); + tokio::spawn(async move { + for metrics in runtime_monitor.intervals() { + collect_tokio(&runtime_metrics, metrics); + tokio::time::sleep(frequency).await; + } + }); + + Ok(()) +} + +const TOKIO_PREFIX: &str = "tokio_rt_"; + +fn collect_tokio( + runtime_metrics: &tokio::runtime::RuntimeMetrics, + worker_metrics: tokio_metrics::RuntimeMetrics, +) { + gauge!( + format!("{}num_workers", TOKIO_PREFIX), + runtime_metrics.num_workers() as f64 + ); + gauge!( + format!("{}num_blocking_threads", TOKIO_PREFIX), + runtime_metrics.num_blocking_threads() as f64 + ); + gauge!( + format!("{}active_tasks_count", TOKIO_PREFIX), + runtime_metrics.active_tasks_count() as f64 + ); + gauge!( + format!("{}num_idle_blocking_threads", TOKIO_PREFIX), + runtime_metrics.num_idle_blocking_threads() as f64 + ); + gauge!( + format!("{}blocking_queue_depth", TOKIO_PREFIX), + runtime_metrics.blocking_queue_depth() as f64 + ); + gauge!( + format!("{}total_park_count", TOKIO_PREFIX), + worker_metrics.total_park_count as f64 + ); + gauge!( + format!("{}max_park_count", TOKIO_PREFIX), + worker_metrics.max_park_count as f64 + ); + gauge!( + format!("{}min_park_count", TOKIO_PREFIX), + worker_metrics.min_park_count as f64 + ); + gauge!( + format!("{}mean_poll_duration", TOKIO_PREFIX), + worker_metrics.mean_poll_duration.as_secs_f64() + ); + gauge!( + format!("{}mean_poll_duration_worker_min", TOKIO_PREFIX), + worker_metrics.mean_poll_duration_worker_min.as_secs_f64() + ); + gauge!( + format!("{}mean_poll_duration_worker_max", TOKIO_PREFIX), + worker_metrics.mean_poll_duration_worker_max.as_secs_f64() + ); + gauge!( + format!("{}total_noop_count", TOKIO_PREFIX), + worker_metrics.total_noop_count as f64, + ); + gauge!( + format!("{}max_noop_count", TOKIO_PREFIX), + worker_metrics.max_noop_count as f64, + ); + gauge!( + format!("{}min_noop_count", TOKIO_PREFIX), + worker_metrics.min_noop_count as f64, + ); + gauge!( + format!("{}total_steal_count", TOKIO_PREFIX), + worker_metrics.total_steal_count as f64, + ); + gauge!( + format!("{}max_steal_count", TOKIO_PREFIX), + worker_metrics.max_steal_count as f64, + ); + gauge!( + format!("{}min_steal_count", TOKIO_PREFIX), + worker_metrics.min_steal_count as f64, + ); + gauge!( + format!("{}total_steal_operations", TOKIO_PREFIX), + worker_metrics.total_steal_operations as f64, + ); + gauge!( + format!("{}max_steal_operations", TOKIO_PREFIX), + worker_metrics.max_steal_operations as f64, + ); + gauge!( + format!("{}min_steal_operations", TOKIO_PREFIX), + worker_metrics.min_steal_operations as f64, + ); + gauge!( + format!("{}num_remote_schedules", TOKIO_PREFIX), + worker_metrics.num_remote_schedules as f64, + ); + gauge!( + format!("{}total_local_schedule_count", TOKIO_PREFIX), + worker_metrics.total_local_schedule_count as f64, + ); + gauge!( + format!("{}max_local_schedule_count", TOKIO_PREFIX), + worker_metrics.max_local_schedule_count as f64, + ); + gauge!( + format!("{}min_local_schedule_count", TOKIO_PREFIX), + worker_metrics.min_local_schedule_count as f64, + ); + gauge!( + format!("{}total_overflow_count", TOKIO_PREFIX), + worker_metrics.total_overflow_count as f64, + ); + gauge!( + format!("{}max_overflow_count", TOKIO_PREFIX), + worker_metrics.max_overflow_count as f64, + ); + gauge!( + format!("{}min_overflow_count", TOKIO_PREFIX), + worker_metrics.min_overflow_count as f64, + ); + gauge!( + format!("{}total_polls_count", TOKIO_PREFIX), + worker_metrics.total_polls_count as f64, + ); + gauge!( + format!("{}max_polls_count", TOKIO_PREFIX), + worker_metrics.max_polls_count as f64, + ); + gauge!( + format!("{}min_polls_count", TOKIO_PREFIX), + worker_metrics.min_polls_count as f64, + ); + gauge!( + format!("{}total_busy_duration", TOKIO_PREFIX), + worker_metrics.total_busy_duration.as_secs_f64(), + ); + gauge!( + format!("{}max_busy_duration", TOKIO_PREFIX), + worker_metrics.max_busy_duration.as_secs_f64(), + ); + gauge!( + format!("{}min_busy_duration", TOKIO_PREFIX), + worker_metrics.min_busy_duration.as_secs_f64(), + ); + gauge!( + format!("{}injection_queue_depth", TOKIO_PREFIX), + worker_metrics.injection_queue_depth as f64, + ); + gauge!( + format!("{}total_local_queue_depth", TOKIO_PREFIX), + worker_metrics.total_local_queue_depth as f64, + ); + gauge!( + format!("{}max_local_queue_depth", TOKIO_PREFIX), + worker_metrics.max_local_queue_depth as f64, + ); + gauge!( + format!("{}min_local_queue_depth", TOKIO_PREFIX), + worker_metrics.min_local_queue_depth as f64, + ); + gauge!( + format!("{}budget_forced_yield_count", TOKIO_PREFIX), + worker_metrics.budget_forced_yield_count as f64, + ); + gauge!( + format!("{}io_driver_ready_count", TOKIO_PREFIX), + worker_metrics.io_driver_ready_count as f64, + ); +} diff --git a/bin/rundler/src/cli/mod.rs b/bin/rundler/src/cli/mod.rs index 2fe678476..d1651c235 100644 --- a/bin/rundler/src/cli/mod.rs +++ b/bin/rundler/src/cli/mod.rs @@ -16,9 +16,9 @@ use clap::{builder::PossibleValuesParser, Args, Parser, Subcommand}; mod builder; mod json; +mod metrics; mod node; mod pool; -mod prometheus_exporter; mod rpc; mod tracing; @@ -41,8 +41,12 @@ pub async fn run() -> anyhow::Result<()> { tracing::info!("Parsed CLI options: {:#?}", opt); let metrics_addr = format!("{}:{}", opt.metrics.host, opt.metrics.port).parse()?; - prometheus_exporter::initialize(metrics_addr, &opt.metrics.tags) - .context("metrics server should start")?; + metrics::initialize( + opt.metrics.sample_interval_millis, + metrics_addr, + &opt.metrics.tags, + ) + .context("metrics server should start")?; match opt.command { Command::Node(args) => node::run(*args, opt.common).await?, @@ -341,6 +345,16 @@ pub struct MetricsArgs { global = true )] tags: Vec, + + /// Sample interval for sampling metrics + #[arg( + long = "metrics.sample_interval_millis", + name = "metrics.sample_interval_millis", + env = "METRICS_HOST", + default_value = "1000", + global = true + )] + sample_interval_millis: u64, } /// CLI options for logging diff --git a/bin/rundler/src/cli/prometheus_exporter.rs b/bin/rundler/src/cli/prometheus_exporter.rs deleted file mode 100644 index 19c6fbcf5..000000000 --- a/bin/rundler/src/cli/prometheus_exporter.rs +++ /dev/null @@ -1,50 +0,0 @@ -// This file is part of Rundler. -// -// Rundler is free software: you can redistribute it and/or modify it under the -// terms of the GNU Lesser General Public License as published by the Free Software -// Foundation, either version 3 of the License, or (at your option) any later version. -// -// Rundler is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -// See the GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along with Rundler. -// If not, see https://www.gnu.org/licenses/. - -use std::{net::SocketAddr, time::Duration}; - -use itertools::Itertools; -use metrics_exporter_prometheus::PrometheusBuilder; -use metrics_process::Collector; -use metrics_util::layers::{PrefixLayer, Stack}; - -pub fn initialize<'a>( - listen_addr: SocketAddr, - tags: impl IntoIterator, -) -> anyhow::Result<()> { - let mut builder = PrometheusBuilder::new().with_http_listener(listen_addr); - - let tags: Vec<(&str, &str)> = tags - .into_iter() - .filter_map(|t| t.split('=').collect_tuple()) - .collect(); - for (k, v) in tags { - builder = builder.add_global_label(k, v); - } - - let (recorder, exporter) = builder.build()?; - tokio::spawn(exporter); - Stack::new(recorder) - .push(PrefixLayer::new("rundler")) - .install()?; - - tokio::spawn(async { - let collector = Collector::default(); - loop { - collector.collect(); - tokio::time::sleep(Duration::from_secs(1)).await; - } - }); - - Ok(()) -} diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 98dacabe0..c21c452ab 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -11,6 +11,7 @@ anyhow.workspace = true ethers.workspace = true futures.workspace = true rand.workspace = true +reqwest.workspace = true tokio.workspace = true tracing.workspace = true url.workspace = true diff --git a/crates/utils/src/eth.rs b/crates/utils/src/eth.rs index 39cc3c121..cb234b37c 100644 --- a/crates/utils/src/eth.rs +++ b/crates/utils/src/eth.rs @@ -59,7 +59,13 @@ pub fn new_provider( poll_interval: Duration, ) -> anyhow::Result>>> { let parsed_url = Url::parse(url).context("provider url should be valid")?; - let http = Http::new(parsed_url); + + let http_client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(1)) + .build() + .context("failed to build reqwest client")?; + let http = Http::new_with_client(parsed_url, http_client); + let client = RetryClientBuilder::default() // these retries are if the server returns a 429 .rate_limit_retries(10) diff --git a/docs/cli.md b/docs/cli.md index fa3596684..a6349787a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -67,6 +67,8 @@ Options for the metrics server: - env: *METRICS_HOST* - `--metrics.tags`: Tags for metrics in the format `key1=value1,key2=value2,...`. - env: *METRICS_TAGS* +- `--metrics.sample_interval_millis`: Sample interval to use for sampling metrics. default: `1000`. + - env: *METRICS_SAMPLE_INTERVAL_MILLIS* ## Logging Options