From 16f22848d575c1ceb0ab7f2c4c365613a085815c Mon Sep 17 00:00:00 2001 From: njgheorghita Date: Thu, 28 Sep 2023 14:44:50 -0400 Subject: [PATCH] Use global metrics registry --- Cargo.lock | 1 + Cargo.toml | 1 + portalnet/src/metrics/mod.rs | 2 ++ portalnet/src/metrics/overlay.rs | 11 ++---- portalnet/src/metrics/portalnet.rs | 16 +++++++++ portalnet/src/metrics/registry.rs | 12 +++++++ portalnet/src/overlay_service.rs | 58 +++++++++++++++--------------- portalnet/src/storage.rs | 40 +++++++++++---------- portalnet/src/types/messages.rs | 8 ++--- portalnet/tests/overlay.rs | 5 ++- src/bin/purge_db.rs | 9 +++-- src/lib.rs | 7 ++-- 12 files changed, 100 insertions(+), 70 deletions(-) create mode 100644 portalnet/src/metrics/portalnet.rs create mode 100644 portalnet/src/metrics/registry.rs diff --git a/Cargo.lock b/Cargo.lock index 4938b1ea9..ab10e90da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6294,6 +6294,7 @@ dependencies = [ "ethportal-api", "ethportal-peertest", "jsonrpsee", + "lazy_static", "parking_lot 0.11.2", "portalnet", "prometheus_exporter", diff --git a/Cargo.toml b/Cargo.toml index 7705f769e..f4432178d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ eth2_ssz = "0.4.0" ethereum-types = "0.12.1" ethportal-api = { path = "ethportal-api" } jsonrpsee = "0.20.0" +lazy_static = "1.4.0" parking_lot = "0.11.2" portalnet = { path = "portalnet" } prometheus_exporter = "0.8.4" diff --git a/portalnet/src/metrics/mod.rs b/portalnet/src/metrics/mod.rs index 3a0e59570..67c32f937 100644 --- a/portalnet/src/metrics/mod.rs +++ b/portalnet/src/metrics/mod.rs @@ -1,3 +1,5 @@ pub mod labels; pub mod overlay; +pub mod portalnet; +pub mod registry; pub mod storage; diff --git a/portalnet/src/metrics/overlay.rs b/portalnet/src/metrics/overlay.rs index 83966a1b4..ae9ae9bf6 100644 --- a/portalnet/src/metrics/overlay.rs +++ b/portalnet/src/metrics/overlay.rs @@ -5,8 +5,8 @@ use crate::types::messages::{Request, Response}; use prometheus_exporter::{ self, prometheus::{ - default_registry, opts, register_int_counter_vec_with_registry, - register_int_gauge_vec_with_registry, IntCounterVec, IntGaugeVec, Registry, + opts, register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry, + IntCounterVec, IntGaugeVec, Registry, }, }; @@ -19,13 +19,6 @@ pub struct OverlayMetrics { pub validation_count: IntCounterVec, } -impl Default for OverlayMetrics { - fn default() -> Self { - let registry = default_registry(); - Self::new(registry).expect("failed to create default metrics") - } -} - impl OverlayMetrics { pub fn new(registry: &Registry) -> anyhow::Result { let message_count = register_int_counter_vec_with_registry!( diff --git a/portalnet/src/metrics/portalnet.rs b/portalnet/src/metrics/portalnet.rs new file mode 100644 index 000000000..b5e597597 --- /dev/null +++ b/portalnet/src/metrics/portalnet.rs @@ -0,0 +1,16 @@ +use crate::metrics::overlay::OverlayMetrics; +use crate::metrics::storage::StorageMetrics; +use prometheus_exporter::{self, prometheus::Registry}; +pub struct PortalnetMetrics { + pub overlay: OverlayMetrics, + pub storage: StorageMetrics, +} + +impl PortalnetMetrics { + // remove this borrow? + pub fn new(registry: &Registry) -> anyhow::Result { + let overlay = OverlayMetrics::new(registry)?; + let storage = StorageMetrics::new(registry)?; + Ok(Self { overlay, storage }) + } +} diff --git a/portalnet/src/metrics/registry.rs b/portalnet/src/metrics/registry.rs new file mode 100644 index 000000000..6d6ec9372 --- /dev/null +++ b/portalnet/src/metrics/registry.rs @@ -0,0 +1,12 @@ +use crate::metrics::portalnet::PortalnetMetrics; +use lazy_static::lazy_static; +use prometheus_exporter::prometheus::Registry; + +lazy_static! { + pub static ref METRICS_REGISTRY: PortalnetMetrics = initialize_metrics_registry(); +} + +fn initialize_metrics_registry() -> PortalnetMetrics { + let registry = Registry::new(); + PortalnetMetrics::new(®istry).expect("failed to initialize metrics") +} diff --git a/portalnet/src/overlay_service.rs b/portalnet/src/overlay_service.rs index a01b3bcba..ae0a8f182 100644 --- a/portalnet/src/overlay_service.rs +++ b/portalnet/src/overlay_service.rs @@ -2727,29 +2727,28 @@ fn pop_while_ssz_bytes_len_gt(enrs: &mut Vec, max_size: usize) { mod tests { use super::*; + use std::net::SocketAddr; use std::time::Instant; - use rstest::rstest; + use discv5::kbucket::Entry; + use ethereum_types::U256; + use rstest::*; + use serial_test::serial; + use tokio::sync::mpsc::unbounded_channel; + use tokio_test::{assert_pending, assert_ready, task}; + use crate::metrics::registry::METRICS_REGISTRY; use crate::{ discovery::{Discovery, NodeAddress}, overlay::OverlayConfig, storage::{DistanceFunction, MemoryContentStore}, types::messages::PortalnetConfig, }; - use ethportal_api::types::content_key::overlay::IdentityContentKey; use ethportal_api::types::distance::XorMetric; use ethportal_api::types::enr::generate_random_remote_enr; use trin_validation::validator::MockValidator; - use discv5::kbucket::Entry; - use ethereum_types::U256; - use serial_test::serial; - use std::net::SocketAddr; - use tokio::sync::mpsc::unbounded_channel; - use tokio_test::{assert_pending, assert_ready, task}; - macro_rules! poll_command_rx { ($service:ident) => { $service.enter(|cx, mut service| service.command_rx.poll_recv(cx)) @@ -2788,7 +2787,8 @@ mod tests { let peers_to_ping = HashSetDelay::default(); let (command_tx, command_rx) = mpsc::unbounded_channel(); let (response_tx, response_rx) = mpsc::unbounded_channel(); - let metrics = Arc::new(OverlayMetrics::default()); + let metrics = METRICS_REGISTRY.overlay.clone(); + let metrics = Arc::new(metrics); let validator = Arc::new(MockValidator {}); OverlayService { @@ -2895,6 +2895,7 @@ mod tests { service.process_ping(ping, node_id); assert_pending!(poll_command_rx!(service)); + std::mem::drop(service); } #[test_log::test(tokio::test)] @@ -2937,6 +2938,7 @@ mod tests { } _ => panic!(), }; + std::mem::drop(service); } #[test_log::test(tokio::test)] @@ -3172,12 +3174,12 @@ mod tests { // Node has maximum radius, so there should be one offer in the channel. OverlayService::::poke_content( - service.kbuckets.clone(), - service.command_tx.clone(), - content_key, - content, - peer_node_ids, - ); + service.kbuckets.clone(), + service.command_tx.clone(), + content_key, + content, + peer_node_ids, + ); let cmd = assert_ready!(poll_command_rx!(service)); let cmd = cmd.unwrap(); if let OverlayCommand::Request(req) = cmd { @@ -3209,12 +3211,12 @@ mod tests { // No nodes in the routing table, so no commands should be in the channel. OverlayService::::poke_content( - service.kbuckets.clone(), - service.command_tx.clone(), - content_key, - content, - peer_node_ids, - ); + service.kbuckets.clone(), + service.command_tx.clone(), + content_key, + content, + peer_node_ids, + ); assert_pending!(poll_command_rx!(service)); } @@ -3260,12 +3262,12 @@ mod tests { // One offer should be in the channel for the maximum radius node. OverlayService::::poke_content( - service.kbuckets.clone(), - service.command_tx.clone(), - content_key, - content, - peer_node_ids, - ); + service.kbuckets.clone(), + service.command_tx.clone(), + content_key, + content, + peer_node_ids, + ); let cmd = assert_ready!(poll_command_rx!(service)); let cmd = cmd.unwrap(); if let OverlayCommand::Request(req) = cmd { diff --git a/portalnet/src/storage.rs b/portalnet/src/storage.rs index b7d6ca971..28506f087 100644 --- a/portalnet/src/storage.rs +++ b/portalnet/src/storage.rs @@ -8,7 +8,6 @@ use std::{ use anyhow::anyhow; use discv5::enr::NodeId; use ethportal_api::types::portal::PaginateLocalContentInfo; -use prometheus_exporter::{self, prometheus::Registry}; use r2d2::Pool; use r2d2_sqlite::SqliteConnectionManager; use rocksdb::{Options, DB}; @@ -186,11 +185,10 @@ impl PortalStorageConfig { storage_capacity_mb: u64, node_data_dir: PathBuf, node_id: NodeId, - registry: &Registry, + metrics: StorageMetrics, ) -> anyhow::Result { let db = Arc::new(PortalStorage::setup_rocksdb(&node_data_dir)?); let sql_connection_pool = PortalStorage::setup_sql(&node_data_dir)?; - let metrics = StorageMetrics::new(registry)?; Ok(Self { storage_capacity_mb, node_id, @@ -821,11 +819,11 @@ pub mod test { use super::*; use discv5::enr::{CombinedKey, EnrBuilder}; - use prometheus_exporter::prometheus::default_registry; use quickcheck::{quickcheck, QuickCheck, TestResult}; use rand::RngCore; use serial_test::serial; + use crate::metrics::registry::METRICS_REGISTRY; use crate::utils::db::{configure_node_data_dir, setup_temp_dir}; use ethportal_api::types::content_key::overlay::IdentityContentKey; @@ -847,7 +845,7 @@ pub mod test { CAPACITY_MB, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let storage = PortalStorage::new(storage_config, ProtocolId::History)?; @@ -875,7 +873,7 @@ pub mod test { CAPACITY_MB, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let mut storage = PortalStorage::new(storage_config, ProtocolId::History).unwrap(); @@ -903,7 +901,7 @@ pub mod test { CAPACITY_MB, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let mut storage = PortalStorage::new(storage_config, ProtocolId::History)?; @@ -929,7 +927,7 @@ pub mod test { CAPACITY_MB, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let mut storage = PortalStorage::new(storage_config, ProtocolId::History)?; @@ -956,7 +954,7 @@ pub mod test { CAPACITY_MB, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let mut storage = PortalStorage::new(storage_config, ProtocolId::History)?; @@ -977,7 +975,7 @@ pub mod test { 1, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let new_storage = PortalStorage::new(new_storage_config, ProtocolId::History)?; @@ -996,7 +994,7 @@ pub mod test { 0, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let new_storage = PortalStorage::new(new_storage_config, ProtocolId::History)?; @@ -1023,7 +1021,7 @@ pub mod test { min_capacity, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let mut storage = PortalStorage::new(storage_config.clone(), ProtocolId::History)?; @@ -1074,7 +1072,7 @@ pub mod test { CAPACITY_MB, node_data_dir.clone(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let mut storage = PortalStorage::new(storage_config, ProtocolId::History)?; @@ -1093,9 +1091,13 @@ pub mod test { std::mem::drop(storage); // test with increased capacity - let new_storage_config = - PortalStorageConfig::new(2 * CAPACITY_MB, node_data_dir, node_id, default_registry()) - .unwrap(); + let new_storage_config = PortalStorageConfig::new( + 2 * CAPACITY_MB, + node_data_dir, + node_id, + METRICS_REGISTRY.storage.clone(), + ) + .unwrap(); let new_storage = PortalStorage::new(new_storage_config, ProtocolId::History)?; // test that previously set value has not been pruned @@ -1123,7 +1125,7 @@ pub mod test { 0, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let mut storage = PortalStorage::new(storage_config, ProtocolId::History)?; @@ -1151,7 +1153,7 @@ pub mod test { CAPACITY_MB, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let storage = PortalStorage::new(storage_config, ProtocolId::History)?; @@ -1176,7 +1178,7 @@ pub mod test { CAPACITY_MB, temp_dir.path().to_path_buf(), node_id, - default_registry(), + METRICS_REGISTRY.storage.clone(), ) .unwrap(); let mut storage = PortalStorage::new(storage_config, ProtocolId::History).unwrap(); diff --git a/portalnet/src/types/messages.rs b/portalnet/src/types/messages.rs index 0af1bf25f..275f28ae3 100644 --- a/portalnet/src/types/messages.rs +++ b/portalnet/src/types/messages.rs @@ -7,7 +7,6 @@ use std::{ }; use ethereum_types::{H256, U256}; -use prometheus_exporter::prometheus::Registry; use rlp::Encodable; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -18,6 +17,7 @@ use thiserror::Error; use validator::ValidationError; use crate::metrics::overlay::OverlayMetrics; +use crate::metrics::registry::METRICS_REGISTRY; use ethportal_api::types::bootnodes::Bootnodes; use ethportal_api::types::bytes::ByteList; use ethportal_api::types::cli::TrinConfig; @@ -176,15 +176,13 @@ pub struct PortalnetConfig { } impl PortalnetConfig { - pub fn new(trin_config: &TrinConfig, private_key: H256, registry: &Registry) -> Self { + pub fn new(trin_config: &TrinConfig, private_key: H256) -> Self { Self { external_addr: trin_config.external_addr, private_key, listen_port: trin_config.discovery_port, no_stun: trin_config.no_stun, bootnodes: trin_config.bootnodes.clone(), - metrics: OverlayMetrics::new(registry) - .expect("to be able to intialize from default metrics registry"), ..Default::default() } } @@ -201,7 +199,7 @@ impl Default for PortalnetConfig { internal_ip: false, no_stun: false, node_addr_cache_capacity: NODE_ADDR_CACHE_CAPACITY, - metrics: OverlayMetrics::default(), + metrics: METRICS_REGISTRY.overlay.clone(), } } } diff --git a/portalnet/tests/overlay.rs b/portalnet/tests/overlay.rs index 0fa2f8926..ad19ea2be 100644 --- a/portalnet/tests/overlay.rs +++ b/portalnet/tests/overlay.rs @@ -3,7 +3,6 @@ use std::{net::SocketAddr, str::FromStr, sync::Arc}; use discv5::TalkRequest; use parking_lot::RwLock; -use prometheus_exporter::prometheus::default_registry; use tokio::{ sync::{mpsc, mpsc::unbounded_channel}, time::{self, Duration}, @@ -16,7 +15,7 @@ use ethportal_api::types::enr::{Enr, SszEnr}; use ethportal_api::utils::bytes::hex_encode_upper; use portalnet::{ discovery::{Discovery, Discv5UdpSocket}, - metrics::overlay::OverlayMetrics, + metrics::registry::METRICS_REGISTRY, overlay::{OverlayConfig, OverlayProtocol}, storage::{ContentStore, DistanceFunction, MemoryContentStore}, types::messages::{Content, Message, PortalnetConfig, ProtocolId}, @@ -47,7 +46,7 @@ async fn init_overlay( store, protocol, validator, - OverlayMetrics::new(default_registry()).unwrap(), + METRICS_REGISTRY.overlay.clone(), ) .await } diff --git a/src/bin/purge_db.rs b/src/bin/purge_db.rs index c80822ce9..9eb088d68 100644 --- a/src/bin/purge_db.rs +++ b/src/bin/purge_db.rs @@ -12,10 +12,10 @@ use ethportal_api::types::execution::header::HeaderWithProof; use ethportal_api::types::execution::receipts::Receipts; use ethportal_api::utils::bytes::hex_encode; use ethportal_api::HistoryContentKey; +use portalnet::metrics::registry::METRICS_REGISTRY; use portalnet::storage::{PortalStorage, PortalStorageConfig}; use portalnet::types::messages::ProtocolId; use portalnet::utils::db::{configure_node_data_dir, configure_trin_data_dir}; -use prometheus_exporter::prometheus::default_registry; use trin_utils::log::init_tracing_logger; /// @@ -43,7 +43,12 @@ pub fn main() -> Result<()> { // Capacity is 0 since it (eg. for data radius calculation) is irrelevant when only removing data. let capacity = 0; let protocol = ProtocolId::History; - let config = PortalStorageConfig::new(capacity, node_data_dir, node_id, default_registry())?; + let config = PortalStorageConfig::new( + capacity, + node_data_dir, + node_id, + METRICS_REGISTRY.storage.clone(), + )?; let storage = PortalStorage::new(config.clone(), protocol).expect("Failed to create portal storage"); let iter = config.db.iterator(IteratorMode::Start); diff --git a/src/lib.rs b/src/lib.rs index 61fe196d8..4946e3c42 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,6 @@ use std::sync::Arc; -use prometheus_exporter::prometheus::Registry; use rpc::{launch_jsonrpc_server, RpcServerHandle}; use tokio::sync::mpsc; use tokio::sync::RwLock; @@ -12,6 +11,7 @@ use utp_rs::socket::UtpSocket; #[cfg(windows)] use ethportal_api::types::cli::Web3TransportType; use ethportal_api::types::cli::{TrinConfig, BEACON_NETWORK, HISTORY_NETWORK, STATE_NETWORK}; +use portalnet::metrics::registry::METRICS_REGISTRY; use portalnet::{ discovery::{Discovery, Discv5UdpSocket}, events::PortalnetEvents, @@ -49,12 +49,11 @@ pub async fn run_trin( // Initialize prometheus metrics // add metrics registry to configs // or should we create a new one every time? - let metrics_registry = Registry::new(); if let Some(addr) = trin_config.enable_metrics_with_url { prometheus_exporter::start(addr)?; } - let portalnet_config = PortalnetConfig::new(&trin_config, private_key, &metrics_registry); + let portalnet_config = PortalnetConfig::new(&trin_config, private_key); // Initialize base discovery protocol let mut discovery = Discovery::new(portalnet_config.clone())?; @@ -71,7 +70,7 @@ pub async fn run_trin( trin_config.mb.into(), node_data_dir, discovery.local_enr().node_id(), - &metrics_registry, + METRICS_REGISTRY.storage.clone(), )?; // Initialize validation oracle