diff --git a/Cargo.lock b/Cargo.lock index f99b579bfe91..4fa9e83a1736 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4177,6 +4177,7 @@ dependencies = [ "cumulus-relay-chain-interface", "cumulus-relay-chain-rpc-interface", "futures", + "parking_lot 0.12.1", "polkadot-availability-recovery", "polkadot-collator-protocol", "polkadot-core-primitives", @@ -13268,6 +13269,7 @@ dependencies = [ "pallet-transaction-payment-rpc-runtime-api", "parity-db", "parity-scale-codec", + "parking_lot 0.12.1", "polkadot-approval-distribution", "polkadot-availability-bitfield-distribution", "polkadot-availability-distribution", @@ -15728,6 +15730,7 @@ dependencies = [ "array-bytes 4.2.0", "arrayvec 0.7.4", "blake2 0.10.6", + "bytes", "futures", "futures-timer", "libp2p-identity", @@ -15793,6 +15796,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tokio-stream", "tokio-test", "tokio-util", "unsigned-varint", @@ -15848,10 +15852,12 @@ name = "sc-network-gossip" version = "0.10.0-dev" dependencies = [ "ahash 0.8.3", + "async-trait", "futures", "futures-timer", "libp2p", "log", + "parity-scale-codec", "quickcheck", "sc-network", "sc-network-common", diff --git a/cumulus/client/relay-chain-minimal-node/Cargo.toml b/cumulus/client/relay-chain-minimal-node/Cargo.toml index ce76fc5cd6d2..ee93df09ce1a 100644 --- a/cumulus/client/relay-chain-minimal-node/Cargo.toml +++ b/cumulus/client/relay-chain-minimal-node/Cargo.toml @@ -47,4 +47,5 @@ array-bytes = "6.1" tracing = "0.1.37" async-trait = "0.1.73" futures = "0.3.28" +parking_lot = "0.12.1" diff --git a/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs b/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs index a785a9f6f79c..5f5bf338ef99 100644 --- a/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs +++ b/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs @@ -15,7 +15,8 @@ // along with Polkadot. If not, see . use futures::{select, StreamExt}; -use std::sync::Arc; +use parking_lot::Mutex; +use std::{collections::HashMap, sync::Arc}; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use polkadot_collator_protocol::{CollatorProtocolSubsystem, ProtocolSide}; @@ -28,7 +29,7 @@ use polkadot_node_core_chain_api::ChainApiSubsystem; use polkadot_node_core_prospective_parachains::ProspectiveParachainsSubsystem; use polkadot_node_core_runtime_api::RuntimeApiSubsystem; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSet, PeerSetProtocolNames}, request_response::{ v1::{self, AvailableDataFetchingRequest}, v2, IncomingRequestReceiver, ReqProtocolNames, @@ -42,7 +43,7 @@ use polkadot_overseer::{ use polkadot_primitives::CollatorPair; use sc_authority_discovery::Service as AuthorityDiscoveryService; -use sc_network::NetworkStateInfo; +use sc_network::{NetworkStateInfo, NotificationService}; use sc_service::TaskManager; use sc_utils::mpsc::tracing_unbounded; @@ -77,6 +78,8 @@ pub(crate) struct CollatorOverseerGenArgs<'a> { pub req_protocol_names: ReqProtocolNames, /// Peerset protocols name mapping pub peer_set_protocol_names: PeerSetProtocolNames, + /// Notification services for validation/collation protocols. + pub notification_services: HashMap>, } fn build_overseer( @@ -94,6 +97,7 @@ fn build_overseer( collator_pair, req_protocol_names, peer_set_protocol_names, + notification_services, }: CollatorOverseerGenArgs<'_>, ) -> Result< (Overseer, Arc>, OverseerHandle), @@ -101,6 +105,8 @@ fn build_overseer( > { let spawner = SpawnGlue(spawner); let network_bridge_metrics: NetworkBridgeMetrics = Metrics::register(registry)?; + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); + let builder = Overseer::builder() .availability_distribution(DummySubsystem) .availability_recovery(AvailabilityRecoverySubsystem::for_collator( @@ -131,6 +137,8 @@ fn build_overseer( sync_oracle, network_bridge_metrics.clone(), peer_set_protocol_names.clone(), + notification_services, + notification_sinks.clone(), )) .network_bridge_tx(NetworkBridgeTxSubsystem::new( network_service, @@ -138,6 +146,7 @@ fn build_overseer( network_bridge_metrics, req_protocol_names, peer_set_protocol_names, + notification_sinks, )) .provisioner(DummySubsystem) .runtime_api(RuntimeApiSubsystem::new( diff --git a/cumulus/client/relay-chain-minimal-node/src/lib.rs b/cumulus/client/relay-chain-minimal-node/src/lib.rs index 8801f93640c1..d121d2d33567 100644 --- a/cumulus/client/relay-chain-minimal-node/src/lib.rs +++ b/cumulus/client/relay-chain-minimal-node/src/lib.rs @@ -21,7 +21,7 @@ use cumulus_relay_chain_rpc_interface::{RelayChainRpcClient, RelayChainRpcInterf use network::build_collator_network; use polkadot_network_bridge::{peer_sets_info, IsAuthority}; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSet, PeerSetProtocolNames}, request_response::{ v1, v2, IncomingRequest, IncomingRequestReceiver, Protocol, ReqProtocolNames, }, @@ -175,10 +175,13 @@ async fn new_minimal_relay_chain( let peer_set_protocol_names = PeerSetProtocolNames::new(genesis_hash, config.chain_spec.fork_id()); let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No }; - - for config in peer_sets_info(is_authority, &peer_set_protocol_names) { - net_config.add_notification_protocol(config); - } + let notification_services = peer_sets_info(is_authority, &peer_set_protocol_names) + .into_iter() + .map(|(config, (peerset, service))| { + net_config.add_notification_protocol(config); + (peerset, service) + }) + .collect::>>(); let request_protocol_names = ReqProtocolNames::new(genesis_hash, config.chain_spec.fork_id()); let (collation_req_receiver_v1, collation_req_receiver_v2, available_data_req_receiver) = @@ -218,6 +221,7 @@ async fn new_minimal_relay_chain( collator_pair, req_protocol_names: request_protocol_names, peer_set_protocol_names, + notification_services, }; let overseer_handle = diff --git a/cumulus/client/relay-chain-minimal-node/src/network.rs b/cumulus/client/relay-chain-minimal-node/src/network.rs index 813dca47a039..95785063c1ae 100644 --- a/cumulus/client/relay-chain-minimal-node/src/network.rs +++ b/cumulus/client/relay-chain-minimal-node/src/network.rs @@ -26,10 +26,9 @@ use sc_network::{ NetworkService, }; -use sc_network::config::FullNetworkConfiguration; +use sc_network::{config::FullNetworkConfiguration, NotificationService}; use sc_network_common::{role::Roles, sync::message::BlockAnnouncesHandshake}; use sc_service::{error::Error, Configuration, NetworkStarter, SpawnTaskHandle}; -use sc_utils::mpsc::tracing_unbounded; use std::{iter, sync::Arc}; @@ -45,7 +44,7 @@ pub(crate) fn build_collator_network( Error, > { let protocol_id = config.protocol_id(); - let block_announce_config = get_block_announce_proto_config::( + let (block_announce_config, _notification_service) = get_block_announce_proto_config::( protocol_id.clone(), &None, Roles::from(&config.role), @@ -69,8 +68,6 @@ pub(crate) fn build_collator_network( let peer_store_handle = peer_store.handle(); spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); - // RX is not used for anything because syncing is not started for the minimal node - let (tx, _rx) = tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let network_params = sc_network::config::Params:: { role: config.role.clone(), executor: { @@ -86,7 +83,6 @@ pub(crate) fn build_collator_network( protocol_id, metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()), block_announce_config, - tx, }; let network_worker = sc_network::NetworkWorker::new(network_params)?; @@ -150,7 +146,7 @@ fn get_block_announce_proto_config( best_number: NumberFor, best_hash: B::Hash, genesis_hash: B::Hash, -) -> NonDefaultSetConfig { +) -> (NonDefaultSetConfig, Box) { let block_announces_protocol = { let genesis_hash = genesis_hash.as_ref(); if let Some(ref fork_id) = fork_id { @@ -160,12 +156,11 @@ fn get_block_announce_proto_config( } }; - NonDefaultSetConfig { - notifications_protocol: block_announces_protocol.into(), - fallback_names: iter::once(format!("/{}/block-announces/1", protocol_id.as_ref()).into()) - .collect(), - max_notification_size: 1024 * 1024, - handshake: Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( + NonDefaultSetConfig::new( + block_announces_protocol.into(), + iter::once(format!("/{}/block-announces/1", protocol_id.as_ref()).into()).collect(), + 1024 * 1024, + Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( roles, best_number, best_hash, @@ -173,11 +168,11 @@ fn get_block_announce_proto_config( ))), // NOTE: `set_config` will be ignored by `protocol.rs` as the block announcement // protocol is still hardcoded into the peerset. - set_config: SetConfig { + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ) } diff --git a/polkadot/node/network/bridge/src/lib.rs b/polkadot/node/network/bridge/src/lib.rs index 46d4a00faace..ddce99d5c2a8 100644 --- a/polkadot/node/network/bridge/src/lib.rs +++ b/polkadot/node/network/bridge/src/lib.rs @@ -83,6 +83,7 @@ pub(crate) enum WireMessage { ViewUpdate(View), } +#[derive(Debug)] pub(crate) struct PeerData { /// The Latest view sent by the peer. view: View, diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index c264c94cc19b..a9339a5c443c 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -14,23 +14,24 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::{collections::HashSet, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use async_trait::async_trait; -use futures::{prelude::*, stream::BoxStream}; +use parking_lot::Mutex; use parity_scale_codec::Encode; use sc_network::{ - config::parse_addr, multiaddr::Multiaddr, types::ProtocolName, Event as NetworkEvent, - IfDisconnected, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkRequest, - NetworkService, OutboundFailure, ReputationChange, RequestFailure, + config::parse_addr, multiaddr::Multiaddr, types::ProtocolName, IfDisconnected, MessageSink, + NetworkPeers, NetworkRequest, NetworkService, OutboundFailure, ReputationChange, + RequestFailure, }; use polkadot_node_network_protocol::{ - peer_set::{ - CollationVersion, PeerSet, PeerSetProtocolNames, ProtocolVersion, ValidationVersion, - }, + peer_set::{CollationVersion, PeerSet, ProtocolVersion, ValidationVersion}, request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests}, v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, PeerId, }; @@ -44,104 +45,94 @@ const LOG_TARGET: &'static str = "parachain::network-bridge-net"; // Helper function to send a validation v1 message to a list of peers. // Messages are always sent via the main protocol, even legacy protocol messages. pub(crate) fn send_validation_message_v1( - net: &mut impl Network, peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v1 message to peers",); send_message( - net, peers, PeerSet::Validation, ValidationVersion::V1.into(), - peerset_protocol_names, message, metrics, + notification_sinks, ); } // Helper function to send a validation vstaging message to a list of peers. // Messages are always sent via the main protocol, even legacy protocol messages. pub(crate) fn send_validation_message_vstaging( - net: &mut impl Network, peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation vstaging message to peers",); send_message( - net, peers, PeerSet::Validation, ValidationVersion::VStaging.into(), - peerset_protocol_names, message, metrics, + notification_sinks, ); } // Helper function to send a validation v2 message to a list of peers. // Messages are always sent via the main protocol, even legacy protocol messages. pub(crate) fn send_validation_message_v2( - net: &mut impl Network, peers: Vec, - protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Validation, ValidationVersion::V2.into(), - protocol_names, message, metrics, + notification_sinks, ); } // Helper function to send a collation v1 message to a list of peers. // Messages are always sent via the main protocol, even legacy protocol messages. pub(crate) fn send_collation_message_v1( - net: &mut impl Network, peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Collation, CollationVersion::V1.into(), - peerset_protocol_names, message, metrics, + notification_sinks, ); } // Helper function to send a collation v2 message to a list of peers. // Messages are always sent via the main protocol, even legacy protocol messages. pub(crate) fn send_collation_message_v2( - net: &mut impl Network, peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Collation, CollationVersion::V2.into(), - peerset_protocol_names, message, metrics, + notification_sinks, ); } @@ -151,19 +142,19 @@ pub(crate) fn send_collation_message_v2( /// messages that are compatible with the passed peer set, as that is currently not enforced by /// this function. These are messages of type `WireMessage` parameterized on the matching type. fn send_message( - net: &mut impl Network, mut peers: Vec, peer_set: PeerSet, version: ProtocolVersion, - protocol_names: &PeerSetProtocolNames, message: M, metrics: &super::Metrics, + network_notification_sinks: &Arc>>>, ) where M: Encode + Clone, { if peers.is_empty() { return } + let message = { let encoded = message.encode(); metrics.on_notification_sent(peer_set, version, encoded.len(), peers.len()); @@ -171,13 +162,13 @@ fn send_message( encoded }; - // optimization: generate the protocol name once. - let protocol_name = protocol_names.get_name(peer_set, version); + let notification_sinks = network_notification_sinks.lock(); + gum::trace!( target: LOG_TARGET, ?peers, + ?peer_set, ?version, - ?protocol_name, ?message, "Sending message to peers", ); @@ -185,29 +176,26 @@ fn send_message( // optimization: avoid cloning the message for the last peer in the // list. The message payload can be quite large. If the underlying // network used `Bytes` this would not be necessary. + // + // peer may have gotten disconnect by the time `send_message()` is called + // at which point the the sink is not available. let last_peer = peers.pop(); - - // We always send messages on the "main" name even when a negotiated - // fallback is used. The libp2p implementation handles the fallback - // under the hood. - let protocol_name = protocol_names.get_main_name(peer_set); peers.into_iter().for_each(|peer| { - net.write_notification(peer, protocol_name.clone(), message.clone()); + if let Some(sink) = notification_sinks.get(&(peer_set, peer)) { + sink.send_sync_notification(message.clone()); + } }); + if let Some(peer) = last_peer { - net.write_notification(peer, protocol_name, message); + if let Some(sink) = notification_sinks.get(&(peer_set, peer)) { + sink.send_sync_notification(message.clone()); + } } } /// An abstraction over networking for the purposes of this subsystem. #[async_trait] pub trait Network: Clone + Send + 'static { - /// Get a stream of all events occurring on the network. This may include events unrelated - /// to the Polkadot protocol - the user of this function should filter only for events related - /// to the [`VALIDATION_PROTOCOL_NAME`](VALIDATION_PROTOCOL_NAME) - /// or [`COLLATION_PROTOCOL_NAME`](COLLATION_PROTOCOL_NAME) - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent>; - /// Ask the network to keep a substream open with these nodes and not disconnect from them /// until removed from the protocol's peer set. /// Note that `out_peers` setting has no effect on this. @@ -239,16 +227,12 @@ pub trait Network: Clone + Send + 'static { /// Disconnect a given peer from the protocol specified without harming reputation. fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName); - /// Write a notification to a peer on the given protocol. - fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec); + /// Get peer role. + fn peer_role(&self, who: PeerId, handshake: Vec) -> Option; } #[async_trait] impl Network for Arc> { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - NetworkService::event_stream(self, "polkadot-network-bridge").boxed() - } - async fn set_reserved_peers( &mut self, protocol: ProtocolName, @@ -273,10 +257,6 @@ impl Network for Arc> { NetworkService::disconnect_peer(&**self, who, protocol); } - fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec) { - NetworkService::write_notification(&**self, who, protocol, message); - } - async fn start_request( &self, authority_discovery: &mut AD, @@ -348,6 +328,10 @@ impl Network for Arc> { if_disconnected, ); } + + fn peer_role(&self, who: PeerId, handshake: Vec) -> Option { + NetworkService::peer_role(self, who, handshake) + } } /// We assume one `peer_id` per `authority_id`. diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 06be57ead006..40cd167a968b 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -20,11 +20,14 @@ use super::*; use always_assert::never; use bytes::Bytes; -use futures::stream::{BoxStream, StreamExt}; use net_protocol::filter_by_peer_version; use parity_scale_codec::{Decode, DecodeAll}; +use parking_lot::Mutex; -use sc_network::Event as NetworkEvent; +use sc_network::{ + service::traits::{NotificationEvent, ValidationResult}, + MessageSink, NotificationService, +}; use sp_consensus::SyncOracle; use polkadot_node_network_protocol::{ @@ -88,6 +91,9 @@ pub struct NetworkBridgeRx { shared: Shared, metrics: Metrics, peerset_protocol_names: PeerSetProtocolNames, + validation_service: Box, + collation_service: Box, + notification_sinks: Arc>>>, } impl NetworkBridgeRx { @@ -102,8 +108,18 @@ impl NetworkBridgeRx { sync_oracle: Box, metrics: Metrics, peerset_protocol_names: PeerSetProtocolNames, + mut notification_services: HashMap>, + notification_sinks: Arc>>>, ) -> Self { let shared = Shared::default(); + + let validation_service = notification_services + .remove(&PeerSet::Validation) + .expect("validation protocol was enabled so `NotificationService` must exist; qed"); + let collation_service = notification_services + .remove(&PeerSet::Collation) + .expect("collation protocol was enabled so `NotificationService` must exist; qed"); + Self { network_service, authority_discovery_service, @@ -111,6 +127,9 @@ impl NetworkBridgeRx { shared, metrics, peerset_protocol_names, + validation_service, + collation_service, + notification_sinks, } } } @@ -121,444 +140,563 @@ where Net: Network + Sync, AD: validator_discovery::AuthorityDiscovery + Clone + Sync, { - fn start(mut self, ctx: Context) -> SpawnedSubsystem { - // The stream of networking events has to be created at initialization, otherwise the - // networking might open connections before the stream of events has been grabbed. - let network_stream = self.network_service.event_stream(); - + fn start(self, ctx: Context) -> SpawnedSubsystem { // Swallow error because failure is fatal to the node and we log with more precision // within `run_network`. - let future = run_network_in(self, ctx, network_stream) + let future = run_network_in(self, ctx) .map_err(|e| SubsystemError::with_origin("network-bridge", e)) .boxed(); SpawnedSubsystem { name: "network-bridge-rx-subsystem", future } } } -async fn handle_network_messages( - mut sender: impl overseer::NetworkBridgeRxSenderTrait, - mut network_service: impl Network, - network_stream: BoxStream<'static, NetworkEvent>, - mut authority_discovery_service: AD, - metrics: Metrics, - shared: Shared, - peerset_protocol_names: PeerSetProtocolNames, -) -> Result<(), Error> -where +/// Handle notification event received over the validation protocol. +async fn handle_validation_message( + event: NotificationEvent, + network_service: &mut impl Network, + sender: &mut impl overseer::NetworkBridgeRxSenderTrait, + authority_discovery_service: &mut AD, + metrics: &Metrics, + shared: &Shared, + peerset_protocol_names: &PeerSetProtocolNames, + notification_service: &mut Box, + notification_sinks: &mut Arc>>>, +) where AD: validator_discovery::AuthorityDiscovery + Send, { - let mut network_stream = network_stream.fuse(); - loop { - match network_stream.next().await { - None => return Err(Error::EventStreamConcluded), - Some(NetworkEvent::Dht(_)) => {}, - Some(NetworkEvent::NotificationStreamOpened { - remote: peer, - protocol, - role, - negotiated_fallback, - received_handshake: _, - }) => { - let role = ObservedRole::from(role); - let (peer_set, version) = { - let (peer_set, version) = - match peerset_protocol_names.try_get_protocol(&protocol) { - None => continue, - Some(p) => p, - }; - - if let Some(fallback) = negotiated_fallback { - match peerset_protocol_names.try_get_protocol(&fallback) { - None => { + match event { + NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx, .. } => { + // only accept peers whose role can be determined + let result = network_service + .peer_role(peer, handshake) + .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); + let _ = result_tx.send(result); + }, + NotificationEvent::NotificationStreamOpened { + peer, + handshake, + negotiated_fallback, + .. + } => { + let role = match network_service.peer_role(peer, handshake) { + Some(role) => ObservedRole::from(role), + None => { + gum::debug!( + target: LOG_TARGET, + ?peer, + "Failed to determine peer role for validation protocol", + ); + return + }, + }; + + let (peer_set, version) = { + let (peer_set, version) = + (PeerSet::Validation, PeerSet::Validation.get_main_version()); + + if let Some(fallback) = negotiated_fallback { + match peerset_protocol_names.try_get_protocol(&fallback) { + None => { + gum::debug!( + target: LOG_TARGET, + fallback = &*fallback, + ?peer, + peerset = ?peer_set, + "Unknown fallback", + ); + + return + }, + Some((p2, v2)) => { + if p2 != peer_set { gum::debug!( target: LOG_TARGET, fallback = &*fallback, - ?peer, - ?peer_set, - "Unknown fallback", + fallback_peerset = ?p2, + peerset = ?peer_set, + "Fallback mismatched peer-set", ); - continue - }, - Some((p2, v2)) => { - if p2 != peer_set { - gum::debug!( - target: LOG_TARGET, - fallback = &*fallback, - fallback_peerset = ?p2, - protocol = &*protocol, - peerset = ?peer_set, - "Fallback mismatched peer-set", - ); - - continue - } - - (p2, v2) - }, - } - } else { - (peer_set, version) - } - }; - - gum::debug!( - target: LOG_TARGET, - action = "PeerConnected", - peer_set = ?peer_set, - version = %version, - peer = ?peer, - role = ?role - ); - - let local_view = { - let mut shared = shared.0.lock(); - let peer_map = match peer_set { - PeerSet::Validation => &mut shared.validation_peers, - PeerSet::Collation => &mut shared.collation_peers, - }; + return + } - match peer_map.entry(peer) { - hash_map::Entry::Occupied(_) => continue, - hash_map::Entry::Vacant(vacant) => { - vacant.insert(PeerData { view: View::default(), version }); + (p2, v2) }, } + } else { + (peer_set, version) + } + }; + // store the notification sink to `notification_sinks` so both `NetworkBridgeRx` + // and `NetworkBridgeTx` can send messages to the peer. + match notification_service.message_sink(&peer) { + Some(sink) => { + notification_sinks.lock().insert((peer_set, peer), sink); + }, + None => { + gum::warn!( + target: LOG_TARGET, + peerset = ?peer_set, + version = %version, + ?peer, + ?role, + "Message sink not available for peer", + ); + return + }, + } - metrics.on_peer_connected(peer_set, version); - metrics.note_peer_count(peer_set, version, peer_map.len()); - - shared.local_view.clone().unwrap_or(View::default()) - }; - - let maybe_authority = - authority_discovery_service.get_authority_ids_by_peer_id(peer).await; - - match peer_set { - PeerSet::Validation => { - dispatch_validation_events_to_all( - vec![ - NetworkBridgeEvent::PeerConnected( - peer, - role, - version, - maybe_authority, - ), - NetworkBridgeEvent::PeerViewChange(peer, View::default()), - ], - &mut sender, - &metrics, - ) - .await; - - match ValidationVersion::try_from(version) - .expect("try_get_protocol has already checked version is known; qed") - { - ValidationVersion::V1 => send_validation_message_v1( - &mut network_service, - vec![peer], - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - ValidationVersion::VStaging => send_validation_message_vstaging( - &mut network_service, - vec![peer], - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - ValidationVersion::V2 => send_validation_message_v2( - &mut network_service, - vec![peer], - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - } - }, - PeerSet::Collation => { - dispatch_collation_events_to_all( - vec![ - NetworkBridgeEvent::PeerConnected( - peer, - role, - version, - maybe_authority, - ), - NetworkBridgeEvent::PeerViewChange(peer, View::default()), - ], - &mut sender, - ) - .await; - - match CollationVersion::try_from(version) - .expect("try_get_protocol has already checked version is known; qed") - { - CollationVersion::V1 => send_collation_message_v1( - &mut network_service, - vec![peer], - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - CollationVersion::V2 => send_collation_message_v2( - &mut network_service, - vec![peer], - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - } + gum::debug!( + target: LOG_TARGET, + action = "PeerConnected", + peer_set = ?peer_set, + version = %version, + peer = ?peer, + role = ?role + ); + + let local_view = { + let mut shared = shared.0.lock(); + let peer_map = &mut shared.validation_peers; + + match peer_map.entry(peer) { + hash_map::Entry::Occupied(_) => return, + hash_map::Entry::Vacant(vacant) => { + vacant.insert(PeerData { view: View::default(), version }); }, } - }, - Some(NetworkEvent::NotificationStreamClosed { remote: peer, protocol }) => { - let (peer_set, version) = match peerset_protocol_names.try_get_protocol(&protocol) { - None => continue, - Some(peer_set) => peer_set, - }; - gum::debug!( - target: LOG_TARGET, - action = "PeerDisconnected", - peer_set = ?peer_set, - peer = ?peer - ); + metrics.on_peer_connected(peer_set, version); + metrics.note_peer_count(peer_set, version, peer_map.len()); - let was_connected = { - let mut shared = shared.0.lock(); - let peer_map = match peer_set { - PeerSet::Validation => &mut shared.validation_peers, - PeerSet::Collation => &mut shared.collation_peers, - }; + shared.local_view.clone().unwrap_or(View::default()) + }; - let w = peer_map.remove(&peer).is_some(); + let maybe_authority = + authority_discovery_service.get_authority_ids_by_peer_id(peer).await; - metrics.on_peer_disconnected(peer_set, version); - metrics.note_peer_count(peer_set, version, peer_map.len()); + dispatch_validation_events_to_all( + vec![ + NetworkBridgeEvent::PeerConnected(peer, role, version, maybe_authority), + NetworkBridgeEvent::PeerViewChange(peer, View::default()), + ], + sender, + &metrics, + ) + .await; - w - }; + match ValidationVersion::try_from(version) + .expect("try_get_protocol has already checked version is known; qed") + { + ValidationVersion::V1 => send_validation_message_v1( + vec![peer], + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + ValidationVersion::VStaging => send_validation_message_vstaging( + vec![peer], + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + ValidationVersion::V2 => send_validation_message_v2( + vec![peer], + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + } + }, + NotificationEvent::NotificationStreamClosed { peer } => { + let (peer_set, version) = (PeerSet::Validation, PeerSet::Validation.get_main_version()); - if was_connected && version == peer_set.get_main_version() { - match peer_set { - PeerSet::Validation => - dispatch_validation_event_to_all( - NetworkBridgeEvent::PeerDisconnected(peer), - &mut sender, - &metrics, - ) - .await, - PeerSet::Collation => - dispatch_collation_event_to_all( - NetworkBridgeEvent::PeerDisconnected(peer), - &mut sender, - ) - .await, - } + gum::debug!( + target: LOG_TARGET, + action = "PeerDisconnected", + ?peer_set, + ?peer + ); + + let was_connected = { + let mut shared = shared.0.lock(); + let peer_map = &mut shared.validation_peers; + + let w = peer_map.remove(&peer).is_some(); + + metrics.on_peer_disconnected(peer_set, version); + metrics.note_peer_count(peer_set, version, peer_map.len()); + + w + }; + + notification_sinks.lock().remove(&(peer_set, peer)); + + if was_connected && version == peer_set.get_main_version() { + dispatch_validation_event_to_all( + NetworkBridgeEvent::PeerDisconnected(peer), + sender, + &metrics, + ) + .await; + } + }, + NotificationEvent::NotificationReceived { peer, notification } => { + let expected_versions = { + let mut versions = PerPeerSet::>::default(); + let shared = shared.0.lock(); + + if let Some(peer_data) = shared.validation_peers.get(&peer) { + versions[PeerSet::Validation] = Some(peer_data.version); } - }, - Some(NetworkEvent::NotificationsReceived { remote, messages }) => { - let expected_versions = { - let mut versions = PerPeerSet::>::default(); - let shared = shared.0.lock(); - if let Some(peer_data) = shared.validation_peers.get(&remote) { - versions[PeerSet::Validation] = Some(peer_data.version); - } - if let Some(peer_data) = shared.collation_peers.get(&remote) { - versions[PeerSet::Collation] = Some(peer_data.version); - } + versions + }; - versions + gum::trace!( + target: LOG_TARGET, + action = "PeerMessage", + peerset = ?PeerSet::Validation, + ?peer, + ); + + let (events, reports) = + if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V1.into()) { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Validation] == + Some(ValidationVersion::V2.into()) + { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Validation] == + Some(ValidationVersion::VStaging.into()) + { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else { + gum::warn!( + target: LOG_TARGET, + version = ?expected_versions[PeerSet::Validation], + "Major logic bug. Peer somehow has unsupported validation protocol version." + ); + + never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); + + // If a peer somehow triggers this, we'll disconnect them + // eventually. + (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) }; - // non-decoded, but version-checked validation messages. - let v_messages: Result, _> = messages - .iter() - .filter_map(|(protocol, msg_bytes)| { - // version doesn't matter because we always receive on the 'correct' - // protocol name, not the negotiated fallback. - let (peer_set, version) = - peerset_protocol_names.try_get_protocol(protocol)?; - gum::trace!( - target: LOG_TARGET, - ?peer_set, - ?protocol, - ?version, - "Received notification" - ); + for report in reports { + network_service.report_peer(peer, report.into()); + } - if peer_set == PeerSet::Validation { - if expected_versions[PeerSet::Validation].is_none() { - return Some(Err(UNCONNECTED_PEERSET_COST)) - } + dispatch_validation_events_to_all(events, sender, &metrics).await; + }, + } +} + +/// Handle notification event received over the collation protocol. +async fn handle_collation_message( + event: NotificationEvent, + network_service: &mut impl Network, + sender: &mut impl overseer::NetworkBridgeRxSenderTrait, + authority_discovery_service: &mut AD, + metrics: &Metrics, + shared: &Shared, + peerset_protocol_names: &PeerSetProtocolNames, + notification_service: &mut Box, + notification_sinks: &mut Arc>>>, +) where + AD: validator_discovery::AuthorityDiscovery + Send, +{ + match event { + NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx, .. } => { + // only accept peers whose role can be determined + let result = network_service + .peer_role(peer, handshake) + .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); + let _ = result_tx.send(result); + }, + NotificationEvent::NotificationStreamOpened { + peer, + handshake, + negotiated_fallback, + .. + } => { + let role = match network_service.peer_role(peer, handshake) { + Some(role) => ObservedRole::from(role), + None => { + gum::debug!( + target: LOG_TARGET, + ?peer, + "Failed to determine peer role for validation protocol", + ); + return + }, + }; + + let (peer_set, version) = { + let (peer_set, version) = + (PeerSet::Collation, PeerSet::Collation.get_main_version()); + + if let Some(fallback) = negotiated_fallback { + match peerset_protocol_names.try_get_protocol(&fallback) { + None => { + gum::debug!( + target: LOG_TARGET, + fallback = &*fallback, + ?peer, + ?peer_set, + "Unknown fallback", + ); + + return + }, + Some((p2, v2)) => { + if p2 != peer_set { + gum::debug!( + target: LOG_TARGET, + fallback = &*fallback, + fallback_peerset = ?p2, + peerset = ?peer_set, + "Fallback mismatched peer-set", + ); - Some(Ok(msg_bytes.clone())) - } else { - None - } - }) - .collect(); + return + } - let v_messages = match v_messages { - Err(rep) => { - gum::debug!(target: LOG_TARGET, action = "ReportPeer"); - network_service.report_peer(remote, rep.into()); + (p2, v2) + }, + } + } else { + (peer_set, version) + } + }; + + // store the notification sink to `notification_sinks` so both `NetworkBridgeRx` + // and `NetworkBridgeTx` can send messages to the peer. + match notification_service.message_sink(&peer) { + Some(sink) => { + notification_sinks.lock().insert((peer_set, peer), sink); + }, + None => { + gum::warn!( + target: LOG_TARGET, + peer_set = ?peer_set, + version = %version, + peer = ?peer, + role = ?role, + "Message sink not available for peer", + ); + return + }, + } - continue + gum::debug!( + target: LOG_TARGET, + action = "PeerConnected", + peer_set = ?peer_set, + version = %version, + peer = ?peer, + role = ?role + ); + + let local_view = { + let mut shared = shared.0.lock(); + let peer_map = &mut shared.collation_peers; + + match peer_map.entry(peer) { + hash_map::Entry::Occupied(_) => return, + hash_map::Entry::Vacant(vacant) => { + vacant.insert(PeerData { view: View::default(), version }); }, - Ok(v) => v, - }; + } - // non-decoded, but version-checked collation messages. - let c_messages: Result, _> = messages - .iter() - .filter_map(|(protocol, msg_bytes)| { - // version doesn't matter because we always receive on the 'correct' - // protocol name, not the negotiated fallback. - let (peer_set, _version) = - peerset_protocol_names.try_get_protocol(protocol)?; - - if peer_set == PeerSet::Collation { - if expected_versions[PeerSet::Collation].is_none() { - return Some(Err(UNCONNECTED_PEERSET_COST)) - } + metrics.on_peer_connected(peer_set, version); + metrics.note_peer_count(peer_set, version, peer_map.len()); - Some(Ok(msg_bytes.clone())) - } else { - None - } - }) - .collect(); + shared.local_view.clone().unwrap_or(View::default()) + }; - let c_messages = match c_messages { - Err(rep) => { - gum::debug!(target: LOG_TARGET, action = "ReportPeer"); - network_service.report_peer(remote, rep.into()); + let maybe_authority = + authority_discovery_service.get_authority_ids_by_peer_id(peer).await; - continue - }, - Ok(v) => v, - }; + dispatch_collation_events_to_all( + vec![ + NetworkBridgeEvent::PeerConnected(peer, role, version, maybe_authority), + NetworkBridgeEvent::PeerViewChange(peer, View::default()), + ], + sender, + ) + .await; - if v_messages.is_empty() && c_messages.is_empty() { - continue - } + match CollationVersion::try_from(version) + .expect("try_get_protocol has already checked version is known; qed") + { + CollationVersion::V1 => send_collation_message_v1( + vec![peer], + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + CollationVersion::V2 => send_collation_message_v2( + vec![peer], + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + } + }, + NotificationEvent::NotificationStreamClosed { peer } => { + let (peer_set, version) = (PeerSet::Collation, PeerSet::Collation.get_main_version()); - gum::trace!( - target: LOG_TARGET, - action = "PeerMessages", - peer = ?remote, - num_validation_messages = %v_messages.len(), - num_collation_messages = %c_messages.len() - ); + gum::debug!( + target: LOG_TARGET, + action = "PeerDisconnected", + ?peer_set, + ?peer + ); - if !v_messages.is_empty() { - let (events, reports) = if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::V1.into()) - { - handle_peer_messages::( - remote, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - v_messages, - &metrics, - ) - } else if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::V2.into()) - { - handle_peer_messages::( - remote, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - v_messages, - &metrics, - ) - } else if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::VStaging.into()) - { - handle_peer_messages::( - remote, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - v_messages, - &metrics, - ) - } else { - gum::warn!( - target: LOG_TARGET, - version = ?expected_versions[PeerSet::Validation], - "Major logic bug. Peer somehow has unsupported validation protocol version." - ); + let was_connected = { + let mut shared = shared.0.lock(); + let peer_map = &mut shared.collation_peers; - never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); + let w = peer_map.remove(&peer).is_some(); - // If a peer somehow triggers this, we'll disconnect them - // eventually. - (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) - }; + metrics.on_peer_disconnected(peer_set, version); + metrics.note_peer_count(peer_set, version, peer_map.len()); - for report in reports { - network_service.report_peer(remote, report.into()); - } + w + }; - dispatch_validation_events_to_all(events, &mut sender, &metrics).await; + notification_sinks.lock().remove(&(peer_set, peer)); + + if was_connected && version == peer_set.get_main_version() { + dispatch_collation_event_to_all(NetworkBridgeEvent::PeerDisconnected(peer), sender) + .await; + } + }, + NotificationEvent::NotificationReceived { peer, notification } => { + let expected_versions = { + let mut versions = PerPeerSet::>::default(); + let shared = shared.0.lock(); + + if let Some(peer_data) = shared.collation_peers.get(&peer) { + versions[PeerSet::Collation] = Some(peer_data.version); } - if !c_messages.is_empty() { - let (events, reports) = if expected_versions[PeerSet::Collation] == - Some(CollationVersion::V1.into()) - { - handle_peer_messages::( - remote, - PeerSet::Collation, - &mut shared.0.lock().collation_peers, - c_messages, - &metrics, - ) - } else if expected_versions[PeerSet::Collation] == - Some(CollationVersion::V2.into()) - { - handle_peer_messages::( - remote, - PeerSet::Collation, - &mut shared.0.lock().collation_peers, - c_messages, - &metrics, - ) - } else { - gum::warn!( - target: LOG_TARGET, - version = ?expected_versions[PeerSet::Collation], - "Major logic bug. Peer somehow has unsupported collation protocol version." - ); + versions + }; - never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); + gum::trace!( + target: LOG_TARGET, + action = "PeerMessage", + peerset = ?PeerSet::Collation, + ?peer, + ); + + let (events, reports) = + if expected_versions[PeerSet::Collation] == Some(CollationVersion::V1.into()) { + handle_peer_messages::( + peer, + PeerSet::Collation, + &mut shared.0.lock().collation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Collation] == Some(CollationVersion::V2.into()) + { + handle_peer_messages::( + peer, + PeerSet::Collation, + &mut shared.0.lock().collation_peers, + vec![notification.into()], + metrics, + ) + } else { + gum::warn!( + target: LOG_TARGET, + version = ?expected_versions[PeerSet::Collation], + "Major logic bug. Peer somehow has unsupported collation protocol version." + ); - // If a peer somehow triggers this, we'll disconnect them - // eventually. - (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) - }; + never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); - for report in reports { - network_service.report_peer(remote, report.into()); - } + // If a peer somehow triggers this, we'll disconnect them + // eventually. + (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) + }; - dispatch_collation_events_to_all(events, &mut sender).await; - } + for report in reports { + network_service.report_peer(peer, report.into()); + } + + dispatch_collation_events_to_all(events, sender).await; + }, + } +} + +async fn handle_network_messages( + mut sender: impl overseer::NetworkBridgeRxSenderTrait, + mut network_service: impl Network, + mut authority_discovery_service: AD, + metrics: Metrics, + shared: Shared, + peerset_protocol_names: PeerSetProtocolNames, + mut validation_service: Box, + mut collation_service: Box, + mut notification_sinks: Arc>>>, +) -> Result<(), Error> +where + AD: validator_discovery::AuthorityDiscovery + Send, +{ + loop { + futures::select! { + event = validation_service.next_event().fuse() => match event { + Some(event) => handle_validation_message( + event, + &mut network_service, + &mut sender, + &mut authority_discovery_service, + &metrics, + &shared, + &peerset_protocol_names, + &mut validation_service, + &mut notification_sinks, + ).await, + None => return Err(Error::EventStreamConcluded), }, + event = collation_service.next_event().fuse() => match event { + Some(event) => handle_collation_message( + event, + &mut network_service, + &mut sender, + &mut authority_discovery_service, + &metrics, + &shared, + &peerset_protocol_names, + &mut collation_service, + &mut notification_sinks, + ).await, + None => return Err(Error::EventStreamConcluded), + } } } } @@ -593,17 +731,15 @@ where } #[overseer::contextbounds(NetworkBridgeRx, prefix = self::overseer)] -async fn run_incoming_orchestra_signals( +async fn run_incoming_orchestra_signals( mut ctx: Context, - mut network_service: N, mut authority_discovery_service: AD, shared: Shared, sync_oracle: Box, metrics: Metrics, - peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, ) -> Result<(), Error> where - N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, { // This is kept sorted, descending, by block number. @@ -695,13 +831,12 @@ where mode = Mode::Active; update_our_view( - &mut network_service, &mut ctx, &live_heads, &shared, finalized_number, &metrics, - &peerset_protocol_names, + ¬ification_sinks, ); } } @@ -735,7 +870,6 @@ where async fn run_network_in( bridge: NetworkBridgeRx, mut ctx: Context, - network_stream: BoxStream<'static, NetworkEvent>, ) -> Result<(), Error> where N: Network, @@ -748,16 +882,21 @@ where sync_oracle, shared, peerset_protocol_names, + validation_service, + collation_service, + notification_sinks, } = bridge; let (task, network_event_handler) = handle_network_messages( ctx.sender().clone(), network_service.clone(), - network_stream, authority_discovery_service.clone(), metrics.clone(), shared.clone(), peerset_protocol_names.clone(), + validation_service, + collation_service, + notification_sinks.clone(), ) .remote_handle(); @@ -766,12 +905,11 @@ where let orchestra_signal_handler = run_incoming_orchestra_signals( ctx, - network_service, authority_discovery_service, shared, sync_oracle, metrics, - peerset_protocol_names, + notification_sinks, ); futures::pin_mut!(orchestra_signal_handler); @@ -791,17 +929,14 @@ fn construct_view( } #[overseer::contextbounds(NetworkBridgeRx, prefix = self::overseer)] -fn update_our_view( - net: &mut Net, +fn update_our_view( ctx: &mut Context, live_heads: &[ActivatedLeaf], shared: &Shared, finalized_number: BlockNumber, metrics: &Metrics, - peerset_protocol_names: &PeerSetProtocolNames, -) where - Net: Network, -{ + notification_sinks: &Arc>>>, +) { let new_view = construct_view(live_heads.iter().map(|v| v.hash), finalized_number); let (validation_peers, collation_peers) = { @@ -849,43 +984,38 @@ fn update_our_view( filter_by_peer_version(&validation_peers, ValidationVersion::VStaging.into()); send_validation_message_v1( - net, v1_validation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view.clone()), metrics, + notification_sinks, ); send_collation_message_v1( - net, v1_collation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view.clone()), metrics, + notification_sinks, ); send_validation_message_v2( - net, v2_validation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view.clone()), metrics, + notification_sinks, ); send_collation_message_v2( - net, v2_collation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view.clone()), metrics, + notification_sinks, ); send_validation_message_vstaging( - net, vstaging_validation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view.clone()), metrics, + notification_sinks, ); let our_view = OurView::new( diff --git a/polkadot/node/network/bridge/src/rx/tests.rs b/polkadot/node/network/bridge/src/rx/tests.rs index f784e78a7f21..e0b86feb6442 100644 --- a/polkadot/node/network/bridge/src/rx/tests.rs +++ b/polkadot/node/network/bridge/src/rx/tests.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . use super::*; -use futures::{channel::oneshot, executor, stream::BoxStream}; +use futures::{channel::oneshot, executor}; use overseer::jaeger; use polkadot_node_network_protocol::{self as net_protocol, OurView}; use polkadot_node_subsystem::messages::NetworkBridgeEvent; @@ -26,10 +26,13 @@ use parking_lot::Mutex; use std::{ collections::HashSet, sync::atomic::{AtomicBool, Ordering}, - task::Poll, }; -use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; +use sc_network::{ + service::traits::{Direction, MessageSink, NotificationService}, + IfDisconnected, Multiaddr, ObservedRole as SubstrateObservedRole, ProtocolName, + ReputationChange, Roles, +}; use polkadot_node_network_protocol::{ peer_set::PeerSetProtocolNames, @@ -47,9 +50,8 @@ use polkadot_node_subsystem_test_helpers::{ mock::new_leaf, SingleItemSink, SingleItemStream, TestSubsystemContextHandle, }; use polkadot_node_subsystem_util::metered; -use polkadot_primitives::{AuthorityDiscoveryId, CandidateHash, Hash}; +use polkadot_primitives::{AuthorityDiscoveryId, Hash}; -use sc_network::Multiaddr; use sp_keyring::Sr25519Keyring; use crate::{network::Network, validator_discovery::AuthorityDiscovery}; @@ -64,10 +66,9 @@ pub enum NetworkAction { WriteNotification(PeerId, PeerSet, Vec), } -// The subsystem's view of the network - only supports a single call to `event_stream`. +// The subsystem's view of the network. #[derive(Clone)] struct TestNetwork { - net_events: Arc>>>, action_tx: Arc>>, protocol_names: Arc, } @@ -79,37 +80,42 @@ struct TestAuthorityDiscovery; // of `NetworkAction`s. struct TestNetworkHandle { action_rx: metered::UnboundedMeteredReceiver, - net_tx: SingleItemSink, - protocol_names: PeerSetProtocolNames, + validation_tx: SingleItemSink, + collation_tx: SingleItemSink, } fn new_test_network( protocol_names: PeerSetProtocolNames, -) -> (TestNetwork, TestNetworkHandle, TestAuthorityDiscovery) { - let (net_tx, net_rx) = polkadot_node_subsystem_test_helpers::single_item_sink(); +) -> ( + TestNetwork, + TestNetworkHandle, + TestAuthorityDiscovery, + Box, + Box, +) { let (action_tx, action_rx) = metered::unbounded(); + let (validation_tx, validation_rx) = polkadot_node_subsystem_test_helpers::single_item_sink(); + let (collation_tx, collation_rx) = polkadot_node_subsystem_test_helpers::single_item_sink(); + let action_tx = Arc::new(Mutex::new(action_tx)); ( TestNetwork { - net_events: Arc::new(Mutex::new(Some(net_rx))), - action_tx: Arc::new(Mutex::new(action_tx)), + action_tx: action_tx.clone(), protocol_names: Arc::new(protocol_names.clone()), }, - TestNetworkHandle { action_rx, net_tx, protocol_names }, + TestNetworkHandle { action_rx, validation_tx, collation_tx }, TestAuthorityDiscovery, + Box::new(TestNotificationService::new( + PeerSet::Validation, + action_tx.clone(), + validation_rx, + )), + Box::new(TestNotificationService::new(PeerSet::Collation, action_tx, collation_rx)), ) } #[async_trait] impl Network for TestNetwork { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - self.net_events - .lock() - .take() - .expect("Subsystem made more than one call to `event_stream`") - .boxed() - } - async fn set_reserved_peers( &mut self, _protocol: ProtocolName, @@ -143,7 +149,8 @@ impl Network for TestNetwork { } fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName) { - let (peer_set, _) = self.protocol_names.try_get_protocol(&protocol).unwrap(); + let (peer_set, version) = self.protocol_names.try_get_protocol(&protocol).unwrap(); + assert_eq!(version, peer_set.get_main_version()); self.action_tx .lock() @@ -151,13 +158,10 @@ impl Network for TestNetwork { .unwrap(); } - fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec) { - let (peer_set, _) = self.protocol_names.try_get_protocol(&protocol).unwrap(); - - self.action_tx - .lock() - .unbounded_send(NetworkAction::WriteNotification(who, peer_set, message)) - .unwrap(); + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(SubstrateObservedRole::from(role))) } } @@ -201,35 +205,85 @@ impl TestNetworkHandle { peer_set: PeerSet, role: ObservedRole, ) { - let protocol_version = ProtocolVersion::from(protocol_version); - self.send_network_event(NetworkEvent::NotificationStreamOpened { - remote: peer, - protocol: self.protocol_names.get_name(peer_set, protocol_version), - negotiated_fallback: None, - role: role.into(), - received_handshake: vec![], - }) - .await; + fn observed_role_to_handshake(role: &ObservedRole) -> Vec { + match role { + &ObservedRole::Light => Roles::LIGHT.encode(), + &ObservedRole::Authority => Roles::AUTHORITY.encode(), + &ObservedRole::Full => Roles::FULL.encode(), + } + } + + // because of how protocol negotiation works, if two peers support at least one common + // protocol, the protocol is negotiated over the main protocol (`ValidationVersion::V2`) but + // if either one of the peers used a fallback protocol for the negotiation (meaning they + // don't support the main protocol but some older version of it ), `negotiated_fallback` is + // set to that protocol. + let negotiated_fallback = match protocol_version { + ValidationVersion::V2 => None, + ValidationVersion::V1 => match peer_set { + PeerSet::Validation => Some(ProtocolName::from("/polkadot/validation/1")), + PeerSet::Collation => Some(ProtocolName::from("/polkadot/collation/1")), + }, + ValidationVersion::VStaging => match peer_set { + PeerSet::Validation => Some(ProtocolName::from("/polkadot/validation/3")), + PeerSet::Collation => unreachable!(), + }, + }; + + match peer_set { + PeerSet::Validation => { + self.validation_tx + .send(NotificationEvent::NotificationStreamOpened { + peer, + direction: Direction::Inbound, + handshake: observed_role_to_handshake(&role), + negotiated_fallback, + }) + .await + .expect("subsystem concluded early"); + }, + PeerSet::Collation => { + self.collation_tx + .send(NotificationEvent::NotificationStreamOpened { + peer, + direction: Direction::Inbound, + handshake: observed_role_to_handshake(&role), + negotiated_fallback, + }) + .await + .expect("subsystem concluded early"); + }, + } } async fn disconnect_peer(&mut self, peer: PeerId, peer_set: PeerSet) { - self.send_network_event(NetworkEvent::NotificationStreamClosed { - remote: peer, - protocol: self.protocol_names.get_main_name(peer_set), - }) - .await; + match peer_set { + PeerSet::Validation => self + .validation_tx + .send(NotificationEvent::NotificationStreamClosed { peer }) + .await + .expect("subsystem concluded early"), + PeerSet::Collation => self + .collation_tx + .send(NotificationEvent::NotificationStreamClosed { peer }) + .await + .expect("subsystem concluded early"), + } } async fn peer_message(&mut self, peer: PeerId, peer_set: PeerSet, message: Vec) { - self.send_network_event(NetworkEvent::NotificationsReceived { - remote: peer, - messages: vec![(self.protocol_names.get_main_name(peer_set), message.into())], - }) - .await; - } - - async fn send_network_event(&mut self, event: NetworkEvent) { - self.net_tx.send(event).await.expect("subsystem concluded early"); + match peer_set { + PeerSet::Validation => self + .validation_tx + .send(NotificationEvent::NotificationReceived { peer, notification: message }) + .await + .expect("subsystem concluded early"), + PeerSet::Collation => self + .collation_tx + .send(NotificationEvent::NotificationReceived { peer, notification: message }) + .await + .expect("subsystem concluded early"), + } } } @@ -240,6 +294,121 @@ fn assert_network_actions_contains(actions: &[NetworkAction], action: &NetworkAc } } +struct TestNotificationService { + peer_set: PeerSet, + action_tx: Arc>>, + rx: SingleItemStream, +} + +impl std::fmt::Debug for TestNotificationService { + fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Ok(()) + } +} + +impl TestNotificationService { + pub fn new( + peer_set: PeerSet, + action_tx: Arc>>, + rx: SingleItemStream, + ) -> Self { + Self { peer_set, action_tx, rx } + } +} + +struct TestMessageSink { + peer: PeerId, + peer_set: PeerSet, + action_tx: Arc>>, +} + +impl TestMessageSink { + fn new( + peer: PeerId, + peer_set: PeerSet, + action_tx: Arc>>, + ) -> TestMessageSink { + Self { peer, peer_set, action_tx } + } +} + +#[async_trait::async_trait] +impl MessageSink for TestMessageSink { + fn send_sync_notification(&self, notification: Vec) { + self.action_tx + .lock() + .unbounded_send(NetworkAction::WriteNotification( + self.peer, + self.peer_set, + notification, + )) + .unwrap(); + } + + async fn send_async_notification( + &self, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } +} + +#[async_trait::async_trait] +impl NotificationService for TestNotificationService { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { + unimplemented!(); + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + fn try_set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + self.rx.next().await + } + + // Clone [`NotificationService`] + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } + + /// Get protocol name. + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } + + /// Get notification sink of the peer. + fn message_sink(&self, peer: &PeerId) -> Option> { + Some(Box::new(TestMessageSink::new(*peer, self.peer_set, self.action_tx.clone()))) + } +} + #[derive(Clone)] struct TestSyncOracle { is_major_syncing: Arc, @@ -335,10 +504,11 @@ fn test_harness>( let peerset_protocol_names = PeerSetProtocolNames::new(genesis_hash, fork_id); let pool = sp_core::testing::TaskExecutor::new(); - let (mut network, network_handle, discovery) = new_test_network(peerset_protocol_names.clone()); + let (network, network_handle, discovery, validation_service, collation_service) = + new_test_network(peerset_protocol_names.clone()); let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool); - let network_stream = network.event_stream(); + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); let shared = Shared::default(); let bridge = NetworkBridgeRx { @@ -348,9 +518,12 @@ fn test_harness>( sync_oracle, shared: shared.clone(), peerset_protocol_names, + validation_service, + collation_service, + notification_sinks, }; - let network_bridge = run_network_in(bridge, context, network_stream) + let network_bridge = run_network_in(bridge, context) .map_err(|_| panic!("subsystem execution failed")) .map(|_| ()); @@ -942,8 +1115,6 @@ fn relays_collation_protocol_messages() { .await; } - // peer A gets reported for sending a collation message. - let collator_protocol_message = protocol_v1::CollatorProtocolMessage::Declare( Sr25519Keyring::Alice.public().into(), Default::default(), @@ -953,19 +1124,23 @@ fn relays_collation_protocol_messages() { let message_v1 = protocol_v1::CollationProtocol::CollatorProtocol(collator_protocol_message.clone()); - network_handle - .peer_message( - peer_a, - PeerSet::Collation, - WireMessage::ProtocolMessage(message_v1.clone()).encode(), - ) - .await; - - let actions = network_handle.next_network_actions(3).await; - assert_network_actions_contains( - &actions, - &NetworkAction::ReputationChange(peer_a, UNCONNECTED_PEERSET_COST.into()), - ); + // peer A gets reported for sending a collation message. + // NOTE: this is not possible since peer A cannot send + // a collation message if it has not opened a collation protocol + + // network_handle + // .peer_message( + // peer_a, + // PeerSet::Collation, + // WireMessage::ProtocolMessage(message_v1.clone()).encode(), + // ) + // .await; + + // let actions = network_handle.next_network_actions(3).await; + // assert_network_actions_contains( + // &actions, + // &NetworkAction::ReputationChange(peer_a, UNCONNECTED_PEERSET_COST.into()), + // ); // peer B has the message relayed. @@ -1212,7 +1387,7 @@ fn our_view_updates_decreasing_order_and_limited_to_max() { fn network_protocol_versioning_view_update() { let (oracle, handle) = make_sync_oracle(false); test_harness(Box::new(oracle), |test_harness| async move { - let TestHarness { mut network_handle, mut virtual_overseer, .. } = test_harness; + let TestHarness { mut network_handle, mut virtual_overseer, shared } = test_harness; let peer_ids: Vec<_> = (0..4).map(|_| PeerId::random()).collect(); let peers = [ @@ -1231,12 +1406,22 @@ fn network_protocol_versioning_view_update() { handle.await_mode_switch().await; + let mut total_validation_peers = 0; + let mut total_collation_peers = 0; + for &(peer_id, peer_set, version) in &peers { network_handle .connect_peer(peer_id, version, peer_set, ObservedRole::Full) .await; + + match peer_set { + PeerSet::Validation => total_validation_peers += 1, + PeerSet::Collation => total_collation_peers += 1, + } } + await_peer_connections(&shared, total_validation_peers, total_collation_peers).await; + let view = view![head]; let actions = network_handle.next_network_actions(4).await; @@ -1264,15 +1449,19 @@ fn network_protocol_versioning_view_update() { #[test] fn network_protocol_versioning_subsystem_msg() { + use polkadot_primitives::CandidateHash; + use std::task::Poll; + let (oracle, _handle) = make_sync_oracle(false); test_harness(Box::new(oracle), |test_harness| async move { - let TestHarness { mut network_handle, mut virtual_overseer, .. } = test_harness; + let TestHarness { mut network_handle, mut virtual_overseer, shared } = test_harness; let peer = PeerId::random(); network_handle .connect_peer(peer, ValidationVersion::V2, PeerSet::Validation, ObservedRole::Full) .await; + await_peer_connections(&shared, 1, 0).await; // bridge will inform about all connected peers. { diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index 5f222ad59c75..bdcd1574e335 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -32,7 +32,7 @@ use polkadot_node_subsystem::{ /// To be passed to [`FullNetworkConfiguration::add_notification_protocol`](). pub use polkadot_node_network_protocol::peer_set::{peer_sets_info, IsAuthority}; use polkadot_node_network_protocol::request_response::Requests; -use sc_network::ReputationChange; +use sc_network::{MessageSink, ReputationChange}; use crate::validator_discovery; @@ -60,6 +60,7 @@ pub struct NetworkBridgeTx { metrics: Metrics, req_protocol_names: ReqProtocolNames, peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, } impl NetworkBridgeTx { @@ -74,6 +75,7 @@ impl NetworkBridgeTx { metrics: Metrics, req_protocol_names: ReqProtocolNames, peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, ) -> Self { Self { network_service, @@ -81,6 +83,7 @@ impl NetworkBridgeTx { metrics, req_protocol_names, peerset_protocol_names, + notification_sinks, } } } @@ -107,6 +110,7 @@ async fn handle_subsystem_messages( metrics: Metrics, req_protocol_names: ReqProtocolNames, peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, ) -> Result<(), Error> where N: Network, @@ -130,6 +134,7 @@ where &metrics, &req_protocol_names, &peerset_protocol_names, + ¬ification_sinks, ) .await; }, @@ -140,13 +145,14 @@ where #[overseer::contextbounds(NetworkBridgeTx, prefix = self::overseer)] async fn handle_incoming_subsystem_communication( _ctx: &mut Context, - mut network_service: N, + network_service: N, validator_discovery: &mut validator_discovery::Service, mut authority_discovery_service: AD, msg: NetworkBridgeTxMessage, metrics: &Metrics, req_protocol_names: &ReqProtocolNames, peerset_protocol_names: &PeerSetProtocolNames, + notification_sinks: &Arc>>>, ) -> (N, AD) where N: Network, @@ -194,25 +200,22 @@ where match msg { Versioned::V1(msg) => send_validation_message_v1( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::VStaging(msg) => send_validation_message_vstaging( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::V2(msg) => send_validation_message_v2( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), } }, @@ -227,25 +230,22 @@ where for (peers, msg) in msgs { match msg { Versioned::V1(msg) => send_validation_message_v1( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::VStaging(msg) => send_validation_message_vstaging( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::V2(msg) => send_validation_message_v2( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), } } @@ -259,18 +259,16 @@ where match msg { Versioned::V1(msg) => send_collation_message_v1( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), } }, @@ -284,18 +282,16 @@ where for (peers, msg) in msgs { match msg { Versioned::V1(msg) => send_collation_message_v1( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), } } @@ -389,6 +385,7 @@ where metrics, req_protocol_names, peerset_protocol_names, + notification_sinks, } = bridge; handle_subsystem_messages( @@ -398,6 +395,7 @@ where metrics, req_protocol_names, peerset_protocol_names, + notification_sinks, ) .await?; diff --git a/polkadot/node/network/bridge/src/tx/tests.rs b/polkadot/node/network/bridge/src/tx/tests.rs index 1a2d9a7a4240..c3cf0f322f68 100644 --- a/polkadot/node/network/bridge/src/tx/tests.rs +++ b/polkadot/node/network/bridge/src/tx/tests.rs @@ -15,15 +15,18 @@ // along with Polkadot. If not, see . use super::*; -use futures::{executor, stream::BoxStream}; +use futures::executor; use polkadot_node_subsystem_util::TimeoutExt; use async_trait::async_trait; use parking_lot::Mutex; use std::collections::HashSet; -use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; +use sc_network::{ + IfDisconnected, ObservedRole as SubstrateObservedRole, ProtocolName, ReputationChange, Roles, +}; +use parity_scale_codec::DecodeAll; use polkadot_node_network_protocol::{ peer_set::{PeerSetProtocolNames, ValidationVersion}, request_response::{outgoing::Requests, ReqProtocolNames}, @@ -51,10 +54,9 @@ pub enum NetworkAction { WriteNotification(PeerId, PeerSet, Vec), } -// The subsystem's view of the network - only supports a single call to `event_stream`. +// The subsystem's view of the network. #[derive(Clone)] struct TestNetwork { - net_events: Arc>>>, action_tx: Arc>>, peerset_protocol_names: Arc, } @@ -66,37 +68,78 @@ struct TestAuthorityDiscovery; // of `NetworkAction`s. struct TestNetworkHandle { action_rx: metered::UnboundedMeteredReceiver, - net_tx: metered::MeteredSender, - peerset_protocol_names: PeerSetProtocolNames, + _peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, + action_tx: Arc>>, +} + +struct TestMessageSink { + peer: PeerId, + peer_set: PeerSet, + action_tx: Arc>>, +} + +impl TestMessageSink { + fn new( + peer: PeerId, + peer_set: PeerSet, + action_tx: Arc>>, + ) -> TestMessageSink { + Self { peer, peer_set, action_tx } + } +} + +#[async_trait::async_trait] +impl MessageSink for TestMessageSink { + fn send_sync_notification(&self, notification: Vec) { + self.action_tx + .lock() + .unbounded_send(NetworkAction::WriteNotification( + self.peer, + self.peer_set, + notification, + )) + .unwrap(); + } + + async fn send_async_notification( + &self, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } } fn new_test_network( peerset_protocol_names: PeerSetProtocolNames, -) -> (TestNetwork, TestNetworkHandle, TestAuthorityDiscovery) { - let (net_tx, net_rx) = metered::channel(10); +) -> ( + TestNetwork, + TestNetworkHandle, + TestAuthorityDiscovery, + Arc>>>, +) { let (action_tx, action_rx) = metered::unbounded(); + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); + let action_tx = Arc::new(Mutex::new(action_tx)); ( TestNetwork { - net_events: Arc::new(Mutex::new(Some(net_rx))), - action_tx: Arc::new(Mutex::new(action_tx)), + action_tx: action_tx.clone(), peerset_protocol_names: Arc::new(peerset_protocol_names.clone()), }, - TestNetworkHandle { action_rx, net_tx, peerset_protocol_names }, + TestNetworkHandle { + action_rx, + _peerset_protocol_names: peerset_protocol_names, + action_tx, + notification_sinks: notification_sinks.clone(), + }, TestAuthorityDiscovery, + notification_sinks, ) } #[async_trait] impl Network for TestNetwork { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - self.net_events - .lock() - .take() - .expect("Subsystem made more than one call to `event_stream`") - .boxed() - } - async fn set_reserved_peers( &mut self, _protocol: ProtocolName, @@ -130,7 +173,8 @@ impl Network for TestNetwork { } fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName) { - let (peer_set, _) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); + let (peer_set, version) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); + assert_eq!(version, peer_set.get_main_version()); self.action_tx .lock() @@ -138,13 +182,10 @@ impl Network for TestNetwork { .unwrap(); } - fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec) { - let (peer_set, _) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); - - self.action_tx - .lock() - .unbounded_send(NetworkAction::WriteNotification(who, peer_set, message)) - .unwrap(); + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(SubstrateObservedRole::from(role))) } } @@ -174,23 +215,14 @@ impl TestNetworkHandle { async fn connect_peer( &mut self, peer: PeerId, - protocol_version: ValidationVersion, + _protocol_version: ValidationVersion, peer_set: PeerSet, - role: ObservedRole, + _role: ObservedRole, ) { - let protocol_version = ProtocolVersion::from(protocol_version); - self.send_network_event(NetworkEvent::NotificationStreamOpened { - remote: peer, - protocol: self.peerset_protocol_names.get_name(peer_set, protocol_version), - negotiated_fallback: None, - role: role.into(), - received_handshake: vec![], - }) - .await; - } - - async fn send_network_event(&mut self, event: NetworkEvent) { - self.net_tx.send(event).await.expect("subsystem concluded early"); + self.notification_sinks.lock().insert( + (peer_set, peer), + Box::new(TestMessageSink::new(peer, peer_set, self.action_tx.clone())), + ); } } @@ -208,7 +240,8 @@ fn test_harness>(test: impl FnOnce(TestHarne let peerset_protocol_names = PeerSetProtocolNames::new(genesis_hash, fork_id); let pool = sp_core::testing::TaskExecutor::new(); - let (network, network_handle, discovery) = new_test_network(peerset_protocol_names.clone()); + let (network, network_handle, discovery, network_notification_sinks) = + new_test_network(peerset_protocol_names.clone()); let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool); @@ -219,6 +252,7 @@ fn test_harness>(test: impl FnOnce(TestHarne Metrics(None), req_protocol_names, peerset_protocol_names, + network_notification_sinks, ); let network_bridge_out_fut = run_network_out(bridge_out, context) @@ -364,9 +398,9 @@ fn network_protocol_versioning_send() { approval_distribution_message.clone(), ); - // Note that bridge doesn't ensure neither peer's protocol version - // or peer set match the message. - let receivers = vec![peer_ids[0], peer_ids[3]]; + // only `peer_ids[0]` opened the validation protocol v2 + // so only they will be sent a notification + let receivers = vec![peer_ids[0]]; virtual_overseer .send(FromOrchestra::Communication { msg: NetworkBridgeTxMessage::SendValidationMessage( @@ -406,7 +440,9 @@ fn network_protocol_versioning_send() { let msg = protocol_v2::CollationProtocol::CollatorProtocol(collator_protocol_message.clone()); - let receivers = vec![peer_ids[1], peer_ids[2]]; + // only `peer_ids[0]` opened the collation protocol v2 + // so only they will be sent a notification + let receivers = vec![peer_ids[1]]; virtual_overseer .send(FromOrchestra::Communication { diff --git a/polkadot/node/network/bridge/src/validator_discovery.rs b/polkadot/node/network/bridge/src/validator_discovery.rs index 86e861fbc5b5..b11af8a8a089 100644 --- a/polkadot/node/network/bridge/src/validator_discovery.rs +++ b/polkadot/node/network/bridge/src/validator_discovery.rs @@ -169,13 +169,12 @@ mod tests { use crate::network::Network; use async_trait::async_trait; - use futures::stream::BoxStream; use polkadot_node_network_protocol::{ request_response::{outgoing::Requests, ReqProtocolNames}, PeerId, }; use polkadot_primitives::Hash; - use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; + use sc_network::{IfDisconnected, ProtocolName, ReputationChange}; use sp_keyring::Sr25519Keyring; use std::collections::{HashMap, HashSet}; @@ -224,10 +223,6 @@ mod tests { #[async_trait] impl Network for TestNetwork { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - panic!() - } - async fn set_reserved_peers( &mut self, _protocol: ProtocolName, @@ -263,7 +258,11 @@ mod tests { panic!() } - fn write_notification(&self, _: PeerId, _: ProtocolName, _: Vec) { + fn peer_role( + &self, + _peer_id: PeerId, + _handshake: Vec, + ) -> Option { panic!() } } diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index eb483dec9709..7e257d508b5b 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -21,6 +21,7 @@ use polkadot_primitives::Hash; use sc_network::{ config::{NonDefaultSetConfig, SetConfig}, types::ProtocolName, + NotificationService, }; use std::{ collections::{hash_map::Entry, HashMap}, @@ -68,7 +69,7 @@ impl PeerSet { self, is_authority: IsAuthority, peerset_protocol_names: &PeerSetProtocolNames, - ) -> NonDefaultSetConfig { + ) -> (NonDefaultSetConfig, (PeerSet, Box)) { // Networking layer relies on `get_main_name()` being the main name of the protocol // for peersets and connection management. let protocol = peerset_protocol_names.get_main_name(self); @@ -76,39 +77,47 @@ impl PeerSet { let max_notification_size = self.get_max_notification_size(is_authority); match self { - PeerSet::Validation => NonDefaultSetConfig { - notifications_protocol: protocol, - fallback_names, - max_notification_size, - handshake: None, - set_config: SetConfig { - // we allow full nodes to connect to validators for gossip - // to ensure any `MIN_GOSSIP_PEERS` always include reserved peers - // we limit the amount of non-reserved slots to be less - // than `MIN_GOSSIP_PEERS` in total - in_peers: super::MIN_GOSSIP_PEERS as u32 / 2 - 1, - out_peers: super::MIN_GOSSIP_PEERS as u32 / 2 - 1, - reserved_nodes: Vec::new(), - non_reserved_mode: sc_network::config::NonReservedPeerMode::Accept, - }, + PeerSet::Validation => { + let (config, notification_service) = NonDefaultSetConfig::new( + protocol, + fallback_names, + max_notification_size, + None, + SetConfig { + // we allow full nodes to connect to validators for gossip + // to ensure any `MIN_GOSSIP_PEERS` always include reserved peers + // we limit the amount of non-reserved slots to be less + // than `MIN_GOSSIP_PEERS` in total + in_peers: super::MIN_GOSSIP_PEERS as u32 / 2 - 1, + out_peers: super::MIN_GOSSIP_PEERS as u32 / 2 - 1, + reserved_nodes: Vec::new(), + non_reserved_mode: sc_network::config::NonReservedPeerMode::Accept, + }, + ); + + (config, (PeerSet::Validation, notification_service)) }, - PeerSet::Collation => NonDefaultSetConfig { - notifications_protocol: protocol, - fallback_names, - max_notification_size, - handshake: None, - set_config: SetConfig { - // Non-authority nodes don't need to accept incoming connections on this peer - // set: - in_peers: if is_authority == IsAuthority::Yes { 100 } else { 0 }, - out_peers: 0, - reserved_nodes: Vec::new(), - non_reserved_mode: if is_authority == IsAuthority::Yes { - sc_network::config::NonReservedPeerMode::Accept - } else { - sc_network::config::NonReservedPeerMode::Deny + PeerSet::Collation => { + let (config, notification_service) = NonDefaultSetConfig::new( + protocol, + fallback_names, + max_notification_size, + None, + SetConfig { + // Non-authority nodes don't need to accept incoming connections on this + // peer set: + in_peers: if is_authority == IsAuthority::Yes { 100 } else { 0 }, + out_peers: 0, + reserved_nodes: Vec::new(), + non_reserved_mode: if is_authority == IsAuthority::Yes { + sc_network::config::NonReservedPeerMode::Accept + } else { + sc_network::config::NonReservedPeerMode::Deny + }, }, - }, + ); + + (config, (PeerSet::Collation, notification_service)) }, } } @@ -204,7 +213,7 @@ impl IndexMut for PerPeerSet { pub fn peer_sets_info( is_authority: IsAuthority, peerset_protocol_names: &PeerSetProtocolNames, -) -> Vec { +) -> Vec<(NonDefaultSetConfig, (PeerSet, Box))> { PeerSet::iter() .map(|s| s.get_info(is_authority, &peerset_protocol_names)) .collect() @@ -286,7 +295,7 @@ impl From for ProtocolVersion { } /// On the wire protocol name to [`PeerSet`] mapping. -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct PeerSetProtocolNames { protocols: HashMap, names: HashMap<(PeerSet, ProtocolVersion), ProtocolName>, diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index e7a4f4a825c0..19efd1b66c45 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -89,6 +89,7 @@ kvdb = "0.13.0" kvdb-rocksdb = { version = "0.19.0", optional = true } parity-db = { version = "0.4.8", optional = true } codec = { package = "parity-scale-codec", version = "3.6.1" } +parking_lot = "0.12.1" # Polkadot polkadot-core-primitives = { path = "../../core-primitives" } diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index 5069ec467c9a..70159301fc41 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -51,7 +51,8 @@ use { }, polkadot_node_core_dispute_coordinator::Config as DisputeCoordinatorConfig, polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, request_response::ReqProtocolNames, + peer_set::{PeerSet, PeerSetProtocolNames}, + request_response::ReqProtocolNames, }, sc_client_api::BlockBackend, sc_transaction_pool_api::OffchainTransactionPoolFactory, @@ -74,7 +75,7 @@ pub use { #[cfg(feature = "full-node")] use polkadot_node_subsystem::jaeger; -use std::{path::PathBuf, sync::Arc, time::Duration}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; use prometheus_endpoint::Registry; #[cfg(feature = "full-node")] @@ -809,9 +810,9 @@ pub fn new_full( // anything in terms of behaviour, but makes the logs more consistent with the other // Substrate nodes. let grandpa_protocol_name = grandpa::protocol_standard_name(&genesis_hash, &config.chain_spec); - net_config.add_notification_protocol(grandpa::grandpa_peers_set_config( - grandpa_protocol_name.clone(), - )); + let (grandpa_protocol_config, grandpa_notification_service) = + grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); + net_config.add_notification_protocol(grandpa_protocol_config); let beefy_gossip_proto_name = beefy::gossip_protocol_name(&genesis_hash, config.chain_spec.fork_id()); @@ -824,12 +825,17 @@ pub fn new_full( client.clone(), prometheus_registry.clone(), ); - if enable_beefy { - net_config.add_notification_protocol(beefy::communication::beefy_peers_set_config( - beefy_gossip_proto_name.clone(), - )); - net_config.add_request_response_protocol(beefy_req_resp_cfg); - } + let beefy_notification_service = match enable_beefy { + false => None, + true => { + let (beefy_notification_config, beefy_notification_service) = + beefy::communication::beefy_peers_set_config(beefy_gossip_proto_name.clone()); + + net_config.add_notification_protocol(beefy_notification_config); + net_config.add_request_response_protocol(beefy_req_resp_cfg); + Some(beefy_notification_service) + }, + }; // validation/collation protocols are enabled only if `Overseer` is enabled let peerset_protocol_names = @@ -840,13 +846,21 @@ pub fn new_full( // // Collators and parachain full nodes require the collator and validator networking to send // collations and to be able to recover PoVs. - if role.is_authority() || is_parachain_node.is_running_alongside_parachain_node() { - use polkadot_network_bridge::{peer_sets_info, IsAuthority}; - let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No }; - for config in peer_sets_info(is_authority, &peerset_protocol_names) { - net_config.add_notification_protocol(config); - } - } + let notification_services = + if role.is_authority() || is_parachain_node.is_running_alongside_parachain_node() { + use polkadot_network_bridge::{peer_sets_info, IsAuthority}; + let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No }; + + peer_sets_info(is_authority, &peerset_protocol_names) + .into_iter() + .map(|(config, (peerset, service))| { + net_config.add_notification_protocol(config); + (peerset, service) + }) + .collect::>>() + } else { + std::collections::HashMap::new() + }; let req_protocol_names = ReqProtocolNames::new(&genesis_hash, config.chain_spec.fork_id()); @@ -1078,6 +1092,7 @@ pub fn new_full( offchain_transaction_pool_factory: OffchainTransactionPoolFactory::new( transaction_pool.clone(), ), + notification_services, }, ) .map_err(|e| { @@ -1179,13 +1194,15 @@ pub fn new_full( // need a keystore, regardless of which protocol we use below. let keystore_opt = if role.is_authority() { Some(keystore_container.keystore()) } else { None }; - if enable_beefy { + // beefy is enabled if its notification service exists + if let Some(notification_service) = beefy_notification_service { let justifications_protocol_name = beefy_on_demand_justifications_handler.protocol_name(); let network_params = beefy::BeefyNetworkParams { network: network.clone(), sync: sync_service.clone(), gossip_protocol_name: beefy_gossip_proto_name, justifications_protocol_name, + notification_service, _phantom: core::marker::PhantomData::, }; let payload_provider = beefy_primitives::mmr::MmrRootProvider::new(client.clone()); @@ -1265,6 +1282,7 @@ pub fn new_full( prometheus_registry: prometheus_registry.clone(), shared_voter_state, telemetry: telemetry.as_ref().map(|x| x.handle()), + notification_service: grandpa_notification_service, offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(transaction_pool.clone()), }; diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index fd618863eeaa..599563d64549 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -26,7 +26,7 @@ use polkadot_node_core_candidate_validation::Config as CandidateValidationConfig use polkadot_node_core_chain_selection::Config as ChainSelectionConfig; use polkadot_node_core_dispute_coordinator::Config as DisputeCoordinatorConfig; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSet, PeerSetProtocolNames}, request_response::{ v1 as request_v1, v2 as request_v2, IncomingRequestReceiver, ReqProtocolNames, }, @@ -41,15 +41,16 @@ use polkadot_overseer::{ OverseerConnector, OverseerHandle, SpawnGlue, }; +use parking_lot::Mutex; use polkadot_primitives::runtime_api::ParachainHost; use sc_authority_discovery::Service as AuthorityDiscoveryService; use sc_client_api::AuxStore; use sc_keystore::LocalKeystore; -use sc_network::NetworkStateInfo; +use sc_network::{NetworkStateInfo, NotificationService}; use sp_api::ProvideRuntimeApi; use sp_blockchain::HeaderBackend; use sp_consensus_babe::BabeApi; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; pub use polkadot_approval_distribution::ApprovalDistribution as ApprovalDistributionSubsystem; pub use polkadot_availability_bitfield_distribution::BitfieldDistribution as BitfieldDistributionSubsystem; @@ -140,6 +141,8 @@ where pub peerset_protocol_names: PeerSetProtocolNames, /// The offchain transaction pool factory. pub offchain_transaction_pool_factory: OffchainTransactionPoolFactory, + /// Notification services for validation/collation protocols. + pub notification_services: HashMap>, } /// Obtain a prepared `OverseerBuilder`, that is initialized @@ -173,6 +176,7 @@ pub fn prepared_overseer_builder( req_protocol_names, peerset_protocol_names, offchain_transaction_pool_factory, + notification_services, }: OverseerGenArgs, ) -> Result< InitializedOverseerBuilder< @@ -218,6 +222,7 @@ where use polkadot_node_subsystem_util::metrics::Metrics; let metrics = ::register(registry)?; + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); let spawner = SpawnGlue(spawner); @@ -235,6 +240,7 @@ where network_bridge_metrics.clone(), req_protocol_names, peerset_protocol_names.clone(), + notification_sinks.clone(), )) .network_bridge_rx(NetworkBridgeRxSubsystem::new( network_service.clone(), @@ -242,6 +248,8 @@ where Box::new(sync_service.clone()), network_bridge_metrics, peerset_protocol_names, + notification_services, + notification_sinks, )) .availability_distribution(AvailabilityDistributionSubsystem::new( keystore.clone(), diff --git a/substrate/bin/node-template/node/src/service.rs b/substrate/bin/node-template/node/src/service.rs index 403202829241..e69428d81906 100644 --- a/substrate/bin/node-template/node/src/service.rs +++ b/substrate/bin/node-template/node/src/service.rs @@ -163,9 +163,9 @@ pub fn new_full(config: Configuration) -> Result { &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), &config.chain_spec, ); - net_config.add_notification_protocol(sc_consensus_grandpa::grandpa_peers_set_config( - grandpa_protocol_name.clone(), - )); + let (grandpa_protocol_config, grandpa_notification_service) = + sc_consensus_grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); + net_config.add_notification_protocol(grandpa_protocol_config); let warp_sync = Arc::new(sc_consensus_grandpa::warp_proof::NetworkProvider::new( backend.clone(), @@ -316,6 +316,7 @@ pub fn new_full(config: Configuration) -> Result { link: grandpa_link, network, sync: Arc::new(sync_service), + notification_service: grandpa_notification_service, voting_rule: sc_consensus_grandpa::VotingRulesBuilder::default().build(), prometheus_registry, shared_voter_state: SharedVoterState::empty(), diff --git a/substrate/bin/node/cli/src/service.rs b/substrate/bin/node/cli/src/service.rs index 1c71b5a39561..a746de8de844 100644 --- a/substrate/bin/node/cli/src/service.rs +++ b/substrate/bin/node/cli/src/service.rs @@ -370,28 +370,28 @@ pub fn new_full_base( let shared_voter_state = rpc_setup; let auth_disc_publish_non_global_ips = config.network.allow_non_globals_in_dht; let mut net_config = sc_network::config::FullNetworkConfiguration::new(&config.network); - let genesis_hash = client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"); let grandpa_protocol_name = grandpa::protocol_standard_name(&genesis_hash, &config.chain_spec); - net_config.add_notification_protocol(grandpa::grandpa_peers_set_config( - grandpa_protocol_name.clone(), - )); + let (grandpa_protocol_config, grandpa_notification_service) = + grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); + net_config.add_notification_protocol(grandpa_protocol_config); - let statement_handler_proto = sc_network_statement::StatementHandlerPrototype::new( - genesis_hash, - config.chain_spec.fork_id(), - ); - net_config.add_notification_protocol(statement_handler_proto.set_config()); + let (statement_handler_proto, statement_config) = + sc_network_statement::StatementHandlerPrototype::new( + genesis_hash, + config.chain_spec.fork_id(), + ); + net_config.add_notification_protocol(statement_config); let mixnet_protocol_name = sc_mixnet::protocol_name(genesis_hash.as_ref(), config.chain_spec.fork_id()); - if let Some(mixnet_config) = &mixnet_config { - net_config.add_notification_protocol(sc_mixnet::peers_set_config( - mixnet_protocol_name.clone(), - mixnet_config, - )); - } + let mixnet_notification_service = mixnet_config.as_ref().map(|mixnet_config| { + let (config, notification_service) = + sc_mixnet::peers_set_config(mixnet_protocol_name.clone(), mixnet_config); + net_config.add_notification_protocol(config); + notification_service + }); let warp_sync = Arc::new(grandpa::warp_proof::NetworkProvider::new( backend.clone(), @@ -422,6 +422,8 @@ pub fn new_full_base( mixnet_protocol_name, transaction_pool.clone(), Some(keystore_container.keystore()), + mixnet_notification_service + .expect("`NotificationService` exists since mixnet was enabled; qed"), ); task_manager.spawn_handle().spawn("mixnet", None, mixnet); } @@ -590,6 +592,7 @@ pub fn new_full_base( link: grandpa_link, network: network.clone(), sync: Arc::new(sync_service.clone()), + notification_service: grandpa_notification_service, telemetry: telemetry.as_ref().map(|x| x.handle()), voting_rule: grandpa::VotingRulesBuilder::default().build(), prometheus_registry: prometheus_registry.clone(), diff --git a/substrate/client/consensus/beefy/src/communication/mod.rs b/substrate/client/consensus/beefy/src/communication/mod.rs index 10a6071aae65..3827559057dd 100644 --- a/substrate/client/consensus/beefy/src/communication/mod.rs +++ b/substrate/client/consensus/beefy/src/communication/mod.rs @@ -67,10 +67,16 @@ pub(crate) mod beefy_protocol_name { /// For standard protocol name see [`beefy_protocol_name::gossip_protocol_name`]. pub fn beefy_peers_set_config( gossip_protocol_name: sc_network::ProtocolName, -) -> sc_network::config::NonDefaultSetConfig { - let mut cfg = sc_network::config::NonDefaultSetConfig::new(gossip_protocol_name, 1024 * 1024); +) -> (sc_network::config::NonDefaultSetConfig, Box) { + let (mut cfg, notification_service) = sc_network::config::NonDefaultSetConfig::new( + gossip_protocol_name, + Vec::new(), + 1024 * 1024, + None, + Default::default(), + ); cfg.allow_non_reserved(25, 25); - cfg + (cfg, notification_service) } // cost scalars for reporting peers. diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs index 3d104f132508..b3ff11add27e 100644 --- a/substrate/client/consensus/beefy/src/lib.rs +++ b/substrate/client/consensus/beefy/src/lib.rs @@ -38,7 +38,7 @@ use parking_lot::Mutex; use prometheus::Registry; use sc_client_api::{Backend, BlockBackend, BlockchainEvents, FinalityNotifications, Finalizer}; use sc_consensus::BlockImport; -use sc_network::{NetworkRequest, ProtocolName}; +use sc_network::{NetworkRequest, NotificationService, ProtocolName}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork, Syncing as GossipSyncing}; use sp_api::ProvideRuntimeApi; use sp_blockchain::{ @@ -178,6 +178,8 @@ pub struct BeefyNetworkParams { pub network: Arc, /// Syncing service implementing a sync oracle and an event stream for peers. pub sync: Arc, + /// Handle for receiving notification events. + pub notification_service: Box, /// Chain specific BEEFY gossip protocol name. See /// [`communication::beefy_protocol_name::gossip_protocol_name`]. pub gossip_protocol_name: ProtocolName, @@ -243,6 +245,7 @@ pub async fn start_beefy_gadget( let BeefyNetworkParams { network, sync, + notification_service, gossip_protocol_name, justifications_protocol_name, .. @@ -264,6 +267,7 @@ pub async fn start_beefy_gadget( let gossip_engine = GossipEngine::new( network.clone(), sync.clone(), + notification_service, gossip_protocol_name.clone(), gossip_validator.clone(), None, diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs index 3aaa59cbfa1c..3f800166e26a 100644 --- a/substrate/client/consensus/beefy/src/tests.rs +++ b/substrate/client/consensus/beefy/src/tests.rs @@ -72,7 +72,7 @@ use substrate_test_runtime_client::{BlockBuilderExt, ClientExt}; use tokio::time::Duration; const GENESIS_HASH: H256 = H256::zero(); -fn beefy_gossip_proto_name() -> ProtocolName { +pub(crate) fn beefy_gossip_proto_name() -> ProtocolName { gossip_protocol_name(GENESIS_HASH, None) } @@ -371,6 +371,7 @@ async fn voter_init_setup( let mut gossip_engine = sc_network_gossip::GossipEngine::new( net.peer(0).network_service().clone(), net.peer(0).sync_service().clone(), + net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), "/beefy/whatever", gossip_validator, None, @@ -392,6 +393,14 @@ where { let tasks = FuturesUnordered::new(); + let mut notification_services = peers + .iter() + .map(|(peer_id, _, _)| { + let peer = &mut net.peers[*peer_id]; + (*peer_id, peer.take_notification_service(&beefy_gossip_proto_name()).unwrap()) + }) + .collect::>(); + for (peer_id, key, api) in peers.into_iter() { let peer = &net.peers[peer_id]; @@ -409,6 +418,7 @@ where let network_params = crate::BeefyNetworkParams { network: peer.network_service().clone(), sync: peer.sync_service().clone(), + notification_service: notification_services.remove(&peer_id).unwrap(), gossip_protocol_name: beefy_gossip_proto_name(), justifications_protocol_name: on_demand_justif_handler.protocol_name(), _phantom: PhantomData, @@ -1045,7 +1055,25 @@ async fn should_initialize_voter_at_custom_genesis() { net.peer(0).client().as_client().finalize_block(hashes[8], None).unwrap(); // load persistent state - nothing in DB, should init at genesis - let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); + // + // NOTE: code from `voter_init_setup()` is moved here because the new network event system + // doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the + // first `GossipEngine` + let known_peers = Arc::new(Mutex::new(KnownPeers::new())); + let (gossip_validator, _) = GossipValidator::new(known_peers); + let gossip_validator = Arc::new(gossip_validator); + let mut gossip_engine = sc_network_gossip::GossipEngine::new( + net.peer(0).network_service().clone(), + net.peer(0).sync_service().clone(), + net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), + "/beefy/whatever", + gossip_validator, + None, + ); + let (beefy_genesis, best_grandpa) = + wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); + let persisted_state = + load_or_init_voter_state(&*backend, &api, beefy_genesis, best_grandpa, 1).unwrap(); // Test initialization at session boundary. // verify voter initialized with single session starting at block `custom_pallet_genesis` (7) @@ -1075,7 +1103,11 @@ async fn should_initialize_voter_at_custom_genesis() { net.peer(0).client().as_client().finalize_block(hashes[10], None).unwrap(); // load persistent state - state preset in DB, but with different pallet genesis - let new_persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); + // the network state persists and uses the old `GossipEngine` initialized for `peer(0)` + let (beefy_genesis, best_grandpa) = + wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); + let new_persisted_state = + load_or_init_voter_state(&*backend, &api, beefy_genesis, best_grandpa, 1).unwrap(); // verify voter initialized with single session starting at block `new_pallet_genesis` (10) let sessions = new_persisted_state.voting_oracle().sessions(); @@ -1371,7 +1403,7 @@ async fn gossipped_finality_proofs() { let api = Arc::new(TestApi::with_validator_set(&validator_set)); let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect(); - let charlie = &net.peers[2]; + let charlie = &mut net.peers[2]; let known_peers = Arc::new(Mutex::new(KnownPeers::::new())); // Charlie will run just the gossip engine and not the full voter. let (gossip_validator, _) = GossipValidator::new(known_peers); @@ -1384,6 +1416,7 @@ async fn gossipped_finality_proofs() { let mut charlie_gossip_engine = sc_network_gossip::GossipEngine::new( charlie.network_service().clone(), charlie.sync_service().clone(), + charlie.take_notification_service(&beefy_gossip_proto_name()).unwrap(), beefy_gossip_proto_name(), charlie_gossip_validator.clone(), None, diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs index 0eea5647e51c..1fbda9740531 100644 --- a/substrate/client/consensus/beefy/src/worker.rs +++ b/substrate/client/consensus/beefy/src/worker.rs @@ -1145,12 +1145,16 @@ pub(crate) mod tests { let api = Arc::new(TestApi::with_validator_set(&genesis_validator_set)); let network = peer.network_service().clone(); let sync = peer.sync_service().clone(); + let notification_service = peer + .take_notification_service(&crate::tests::beefy_gossip_proto_name()) + .unwrap(); let known_peers = Arc::new(Mutex::new(KnownPeers::new())); let (gossip_validator, gossip_report_stream) = GossipValidator::new(known_peers.clone()); let gossip_validator = Arc::new(gossip_validator); let gossip_engine = GossipEngine::new( network.clone(), sync.clone(), + notification_service, "/beefy/1", gossip_validator.clone(), None, diff --git a/substrate/client/consensus/grandpa/src/communication/mod.rs b/substrate/client/consensus/grandpa/src/communication/mod.rs index 6d9e956b41be..5c7e1276297a 100644 --- a/substrate/client/consensus/grandpa/src/communication/mod.rs +++ b/substrate/client/consensus/grandpa/src/communication/mod.rs @@ -46,7 +46,7 @@ use finality_grandpa::{ Message::{Precommit, Prevote, PrimaryPropose}, }; use parity_scale_codec::{Decode, DecodeAll, Encode}; -use sc_network::{NetworkBlock, NetworkSyncForkRequest, ReputationChange}; +use sc_network::{NetworkBlock, NetworkSyncForkRequest, NotificationService, ReputationChange}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork}; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; use sp_keystore::KeystorePtr; @@ -247,6 +247,7 @@ impl, S: Syncing> NetworkBridge { pub(crate) fn new( service: N, sync: S, + notification_service: Box, config: crate::Config, set_state: crate::environment::SharedVoterSetState, prometheus_registry: Option<&Registry>, @@ -260,6 +261,7 @@ impl, S: Syncing> NetworkBridge { let gossip_engine = Arc::new(Mutex::new(GossipEngine::new( service.clone(), sync.clone(), + notification_service, protocol, validator.clone(), prometheus_registry, diff --git a/substrate/client/consensus/grandpa/src/communication/tests.rs b/substrate/client/consensus/grandpa/src/communication/tests.rs index 4a869d0f5152..b76b1af93dac 100644 --- a/substrate/client/consensus/grandpa/src/communication/tests.rs +++ b/substrate/client/consensus/grandpa/src/communication/tests.rs @@ -24,16 +24,17 @@ use super::{ }; use crate::{communication::grandpa_protocol_name, environment::SharedVoterSetState}; use futures::prelude::*; -use parity_scale_codec::Encode; +use parity_scale_codec::{DecodeAll, Encode}; use sc_network::{ config::{MultiaddrWithPeerId, Role}, event::Event as NetworkEvent, + service::traits::{Direction, MessageSink, NotificationEvent, NotificationService}, types::ProtocolName, Multiaddr, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT as NotificationSender, PeerId, ReputationChange, }; -use sc_network_common::role::ObservedRole; +use sc_network_common::role::{ObservedRole, Roles}; use sc_network_gossip::Validator; use sc_network_sync::{SyncEvent as SyncStreamEvent, SyncEventStream}; use sc_network_test::{Block, Hash}; @@ -123,6 +124,12 @@ impl NetworkPeers for TestNetwork { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(ObservedRole::from(role))) + } } impl NetworkEventStream for TestNetwork { @@ -211,10 +218,70 @@ impl NetworkSyncForkRequest> for TestSync { fn set_sync_fork_request(&self, _peers: Vec, _hash: Hash, _number: NumberFor) {} } +#[derive(Debug)] +pub(crate) struct TestNotificationService { + sender: TracingUnboundedSender, + rx: TracingUnboundedReceiver, +} + +#[async_trait::async_trait] +impl NotificationService for TestNotificationService { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, peer: &PeerId, notification: Vec) { + let _ = self.sender.unbounded_send(Event::WriteNotification(*peer, notification)); + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + fn try_set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + self.rx.next().await + } + + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } + + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } + + fn message_sink(&self, _peer: &PeerId) -> Option> { + unimplemented!(); + } +} + pub(crate) struct Tester { pub(crate) net_handle: super::NetworkBridge, gossip_validator: Arc>, pub(crate) events: TracingUnboundedReceiver, + pub(crate) notification_tx: TracingUnboundedSender, } impl Tester { @@ -279,6 +346,9 @@ fn voter_set_state() -> SharedVoterSetState { // needs to run in a tokio runtime. pub(crate) fn make_test_network() -> (impl Future, TestNetwork) { let (tx, rx) = tracing_unbounded("test", 100_000); + let (notification_tx, notification_rx) = tracing_unbounded("test-notification", 100_000); + + let notification_service = TestNotificationService { rx: notification_rx, sender: tx.clone() }; let net = TestNetwork { sender: tx }; let sync = TestSync {}; @@ -293,14 +363,22 @@ pub(crate) fn make_test_network() -> (impl Future, TestNetwork) } } - let bridge = - super::NetworkBridge::new(net.clone(), sync, config(), voter_set_state(), None, None); + let bridge = super::NetworkBridge::new( + net.clone(), + sync, + Box::new(notification_service), + config(), + voter_set_state(), + None, + None, + ); ( futures::future::ready(Tester { gossip_validator: bridge.validator.clone(), net_handle: bridge, events: rx, + notification_tx, }), net, ) @@ -385,63 +463,62 @@ fn good_commit_leads_to_relay() { let commit_to_send = encoded_commit.clone(); let network_bridge = tester.net_handle.clone(); - // asking for global communication will cause the test network - // to send us an event asking us for a stream. use it to - // send a message. + // `NetworkBridge` will be operational as soon as it's created and it's + // waiting for events from the network. Send it events that inform that + // a notification stream was opened and that a notification was received. + // + // Since each protocol has its own notification stream, events need not be filtered. let sender_id = id; - let send_message = tester.filter_network_events(move |event| match event { - Event::EventStream(sender) => { - // Add the sending peer and send the commit - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: sender_id, - protocol: grandpa_protocol_name::NAME.into(), + + let send_message = async move { + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: sender_id, + direction: Direction::Inbound, negotiated_fallback: None, - role: ObservedRole::Full, - received_handshake: vec![], + handshake: Roles::FULL.encode(), + }, + ); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: sender_id, + notification: commit_to_send.clone(), + }, + ); + + // Add a random peer which will be the recipient of this message + let receiver_id = PeerId::random(); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: receiver_id, + direction: Direction::Inbound, + negotiated_fallback: None, + handshake: Roles::FULL.encode(), + }, + ); + + // Announce its local set being on the current set id through a neighbor + // packet, otherwise it won't be eligible to receive the commit + let _ = { + let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket { + round: Round(round), + set_id: SetId(set_id), + commit_finalized_height: 1, }); - let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: sender_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - commit_to_send.clone().into(), - )], - }); + let msg = gossip::GossipMessage::::Neighbor(update); - // Add a random peer which will be the recipient of this message - let receiver_id = PeerId::random(); - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: receiver_id, - protocol: grandpa_protocol_name::NAME.into(), - negotiated_fallback: None, - role: ObservedRole::Full, - received_handshake: vec![], - }); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: receiver_id, + notification: msg.encode(), + }, + ); + }; - // Announce its local set has being on the current set id through a neighbor - // packet, otherwise it won't be eligible to receive the commit - let _ = { - let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket { - round: Round(round), - set_id: SetId(set_id), - commit_finalized_height: 1, - }); - - let msg = gossip::GossipMessage::::Neighbor(update); - - sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: receiver_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - msg.encode().into(), - )], - }) - }; - - true - }, - _ => false, - }); + tester + } + .boxed(); // when the commit comes in, we'll tell the callback it was good. let handle_commit = commits_in.into_future().map(|(item, _)| match item.unwrap() { @@ -537,31 +614,32 @@ fn bad_commit_leads_to_report() { let commit_to_send = encoded_commit.clone(); let network_bridge = tester.net_handle.clone(); - // asking for global communication will cause the test network - // to send us an event asking us for a stream. use it to - // send a message. + // `NetworkBridge` will be operational as soon as it's created and it's + // waiting for events from the network. Send it events that inform that + // a notification stream was opened and that a notification was received. + // + // Since each protocol has its own notification stream, events need not be filtered. let sender_id = id; - let send_message = tester.filter_network_events(move |event| match event { - Event::EventStream(sender) => { - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: sender_id, - protocol: grandpa_protocol_name::NAME.into(), + + let send_message = async move { + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: sender_id, + direction: Direction::Inbound, negotiated_fallback: None, - role: ObservedRole::Full, - received_handshake: vec![], - }); - let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: sender_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - commit_to_send.clone().into(), - )], - }); + handshake: Roles::FULL.encode(), + }, + ); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: sender_id, + notification: commit_to_send.clone(), + }, + ); - true - }, - _ => false, - }); + tester + } + .boxed(); // when the commit comes in, we'll tell the callback it was bad. let handle_commit = commits_in.into_future().map(|(item, _)| match item.unwrap() { diff --git a/substrate/client/consensus/grandpa/src/lib.rs b/substrate/client/consensus/grandpa/src/lib.rs index a4584e6fc807..b7cfc9f5b601 100644 --- a/substrate/client/consensus/grandpa/src/lib.rs +++ b/substrate/client/consensus/grandpa/src/lib.rs @@ -67,7 +67,7 @@ use sc_client_api::{ BlockchainEvents, CallExecutor, ExecutorProvider, Finalizer, LockImportRun, StorageProvider, }; use sc_consensus::BlockImport; -use sc_network::types::ProtocolName; +use sc_network::{types::ProtocolName, NotificationService}; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; @@ -687,6 +687,8 @@ pub struct GrandpaParams { pub network: N, /// Event stream for syncing-related events. pub sync: S, + /// Handle for interacting with `Notifications`. + pub notification_service: Box, /// A voting rule used to potentially restrict target votes. pub voting_rule: VR, /// The prometheus metrics registry. @@ -707,21 +709,21 @@ pub struct GrandpaParams { /// For standard protocol name see [`crate::protocol_standard_name`]. pub fn grandpa_peers_set_config( protocol_name: ProtocolName, -) -> sc_network::config::NonDefaultSetConfig { +) -> (sc_network::config::NonDefaultSetConfig, Box) { use communication::grandpa_protocol_name; - sc_network::config::NonDefaultSetConfig { - notifications_protocol: protocol_name, - fallback_names: grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(), + sc_network::config::NonDefaultSetConfig::new( + protocol_name, + grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(), // Notifications reach ~256kiB in size at the time of writing on Kusama and Polkadot. - max_notification_size: 1024 * 1024, - handshake: None, - set_config: sc_network::config::SetConfig { + 1024 * 1024, + None, + sc_network::config::SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: sc_network::config::NonReservedPeerMode::Deny, }, - } + ) } /// Run a GRANDPA voter as a task. Provide configuration and a link to a @@ -744,6 +746,7 @@ where link, network, sync, + notification_service, voting_rule, prometheus_registry, shared_voter_state, @@ -770,6 +773,7 @@ where let network = NetworkBridge::new( network, sync, + notification_service, config.clone(), persistent_data.set_state.clone(), prometheus_registry.as_ref(), diff --git a/substrate/client/consensus/grandpa/src/observer.rs b/substrate/client/consensus/grandpa/src/observer.rs index 8541baa822bb..608ff5e46a0e 100644 --- a/substrate/client/consensus/grandpa/src/observer.rs +++ b/substrate/client/consensus/grandpa/src/observer.rs @@ -28,6 +28,7 @@ use futures::prelude::*; use log::{debug, info, warn}; use sc_client_api::backend::Backend; +use sc_network::NotificationService; use sc_telemetry::TelemetryHandle; use sc_utils::mpsc::TracingUnboundedReceiver; use sp_blockchain::HeaderMetadata; @@ -168,6 +169,7 @@ pub fn run_grandpa_observer( link: LinkHalf, network: N, sync: S, + notification_service: Box, ) -> sp_blockchain::Result + Send> where BE: Backend + Unpin + 'static, @@ -189,6 +191,7 @@ where let network = NetworkBridge::new( network, sync, + notification_service, config.clone(), persistent_data.set_state.clone(), None, @@ -414,14 +417,14 @@ mod tests { use futures::executor; - /// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`. Regression - /// test for bug introduced in d4fbb897c and fixed in b7af8b339. + /// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`. + /// Regression test for bug introduced in d4fbb897c and fixed in b7af8b339. /// - /// When polled, `NetworkBridge` forwards reputation change requests from the `GossipValidator` - /// to the underlying `dyn Network`. This test triggers a reputation change by calling - /// `GossipValidator::validate` with an invalid gossip message. After polling the `ObserverWork` - /// which should poll the `NetworkBridge`, the reputation change should be forwarded to the test - /// network. + /// When polled, `NetworkBridge` forwards reputation change requests from the + /// `GossipValidator` to the underlying `dyn Network`. This test triggers a reputation change + /// by calling `GossipValidator::validate` with an invalid gossip message. After polling the + /// `ObserverWork` which should poll the `NetworkBridge`, the reputation change should be + /// forwarded to the test network. #[test] fn observer_work_polls_underlying_network_bridge() { // Create a test network. @@ -463,12 +466,6 @@ mod tests { // validator to the test network. assert!(observer.now_or_never().is_none()); - // Ignore initial event stream request by gossip engine. - match tester.events.next().now_or_never() { - Some(Some(Event::EventStream(_))) => {}, - _ => panic!("expected event stream request"), - }; - assert_matches!(tester.events.next().now_or_never(), Some(Some(Event::Report(_, _)))); }); } diff --git a/substrate/client/consensus/grandpa/src/tests.rs b/substrate/client/consensus/grandpa/src/tests.rs index 644befe98853..7e42c2d45c73 100644 --- a/substrate/client/consensus/grandpa/src/tests.rs +++ b/substrate/client/consensus/grandpa/src/tests.rs @@ -317,6 +317,9 @@ fn initialize_grandpa( (net.peers[peer_id].network_service().clone(), link) }; let sync = net.peers[peer_id].sync_service().clone(); + let notification_service = net.peers[peer_id] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -332,6 +335,7 @@ fn initialize_grandpa( link, network: net_service, sync, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -472,6 +476,9 @@ async fn finalize_3_voters_1_full_observer() { let net_service = net.peers[peer_id].network_service().clone(); let sync = net.peers[peer_id].sync_service().clone(); let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed"); + let notification_service = net.peers[peer_id] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -487,6 +494,7 @@ async fn finalize_3_voters_1_full_observer() { link, network: net_service, sync, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -557,14 +565,17 @@ async fn transition_3_voters_twice_1_full_observer() { for (peer_id, local_key) in all_peers.clone().into_iter().enumerate() { let keystore = create_keystore(local_key); - let (net_service, link, sync) = { - let net = net.lock(); + let (net_service, link, sync, notification_service) = { + let mut net = net.lock(); let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed"); ( net.peers[peer_id].network_service().clone(), link, net.peers[peer_id].sync_service().clone(), + net.peers[peer_id] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(), ) }; @@ -582,6 +593,7 @@ async fn transition_3_voters_twice_1_full_observer() { link, network: net_service, sync, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1025,6 +1037,9 @@ async fn voter_persists_its_votes() { communication::NetworkBridge::new( net.peers[1].network_service().clone(), net.peers[1].sync_service().clone(), + net.peers[1] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(), config.clone(), set_state, None, @@ -1043,6 +1058,9 @@ async fn voter_persists_its_votes() { (net.peers[0].network_service().clone(), link) }; let sync = net.peers[0].sync_service().clone(); + let notification_service = net.peers[0] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -1058,6 +1076,7 @@ async fn voter_persists_its_votes() { link, network: net_service, sync, + notification_service, voting_rule: VotingRulesBuilder::default().build(), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1082,6 +1101,9 @@ async fn voter_persists_its_votes() { net.add_authority_peer(); let net_service = net.peers[2].network_service().clone(); let sync = net.peers[2].sync_service().clone(); + let notification_service = net.peers[2] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); // but we'll reuse the client from the first peer (alice_voter1) // since we want to share the same database, so that we can // read the persisted state after aborting alice_voter1. @@ -1104,6 +1126,7 @@ async fn voter_persists_its_votes() { link, network: net_service, sync, + notification_service, voting_rule: VotingRulesBuilder::default().build(), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1255,6 +1278,9 @@ async fn finalize_3_voters_1_light_observer() { let mut net = GrandpaTestNet::new(TestApi::new(voters), 3, 1); let voters = initialize_grandpa(&mut net, authorities); + let notification_service = net.peers[3] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); let observer = observer::run_grandpa_observer( Config { gossip_duration: TEST_GOSSIP_DURATION, @@ -1269,6 +1295,7 @@ async fn finalize_3_voters_1_light_observer() { net.peers[3].data.lock().take().expect("link initialized at startup; qed"), net.peers[3].network_service().clone(), net.peers[3].sync_service().clone(), + notification_service, ) .unwrap(); net.peer(0).push_blocks(20, false); @@ -1317,6 +1344,10 @@ async fn voter_catches_up_to_latest_round_when_behind() { link, network: net.peer(peer_id).network_service().clone(), sync: net.peer(peer_id).sync_service().clone(), + notification_service: net + .peer(peer_id) + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(), voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1409,6 +1440,7 @@ fn test_environment_with_select_chain( keystore: Option, network_service: N, sync_service: S, + notification_service: Box, select_chain: SC, voting_rule: VR, ) -> TestEnvironment @@ -1433,6 +1465,7 @@ where let network = NetworkBridge::new( network_service.clone(), sync_service, + notification_service, config.clone(), set_state.clone(), None, @@ -1462,6 +1495,7 @@ fn test_environment( keystore: Option, network_service: N, sync_service: S, + notification_service: Box, voting_rule: VR, ) -> TestEnvironment, VR> where @@ -1474,6 +1508,7 @@ where keystore, network_service, sync_service, + notification_service, link.select_chain.clone(), voting_rule, ) @@ -1490,14 +1525,22 @@ async fn grandpa_environment_respects_voting_rules() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let mut notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); // add 21 blocks let hashes = peer.push_blocks(21, false); // create an environment with no voting rule restrictions - let unrestricted_env = - test_environment(&link, None, network_service.clone(), sync_service.clone(), ()); + let unrestricted_env = test_environment( + &link, + None, + network_service.clone(), + sync_service.clone(), + notification_service.clone().unwrap(), + (), + ); // another with 3/4 unfinalized chain voting rule restriction let three_quarters_env = test_environment( @@ -1505,6 +1548,7 @@ async fn grandpa_environment_respects_voting_rules() { None, network_service.clone(), sync_service.clone(), + notification_service.clone().unwrap(), voting_rule::ThreeQuartersOfTheUnfinalizedChain, ); @@ -1515,6 +1559,7 @@ async fn grandpa_environment_respects_voting_rules() { None, network_service.clone(), sync_service, + notification_service, VotingRulesBuilder::default().build(), ); @@ -1608,6 +1653,8 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let client = peer.client().as_client().clone(); let select_chain = MockSelectChain::default(); @@ -1622,6 +1669,7 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() { None, network_service.clone(), sync_service, + notification_service, select_chain.clone(), voting_rule::BeforeBestBlockBy(5), ); @@ -1669,6 +1717,8 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let client = peer.client().as_client().clone(); let select_chain = MockSelectChain::default(); @@ -1678,6 +1728,7 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ None, network_service.clone(), sync_service.clone(), + notification_service, select_chain.clone(), voting_rule.clone(), ); @@ -1780,11 +1831,19 @@ async fn grandpa_environment_never_overwrites_round_voter_state() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let keystore = create_keystore(peers[0]); - let environment = - test_environment(&link, Some(keystore), network_service.clone(), sync_service, ()); + let environment = test_environment( + &link, + Some(keystore), + network_service.clone(), + sync_service, + notification_service, + (), + ); let round_state = || finality_grandpa::round::State::genesis(Default::default()); let base = || Default::default(); @@ -2012,9 +2071,18 @@ async fn grandpa_environment_doesnt_send_equivocation_reports_for_itself() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let keystore = create_keystore(alice); - test_environment(&link, Some(keystore), network_service.clone(), sync_service, ()) + test_environment( + &link, + Some(keystore), + network_service.clone(), + sync_service, + notification_service, + (), + ) }; let signed_prevote = { diff --git a/substrate/client/executor/wasmtime/src/tests.rs b/substrate/client/executor/wasmtime/src/tests.rs index e185754b0769..1c06da1e3c14 100644 --- a/substrate/client/executor/wasmtime/src/tests.rs +++ b/substrate/client/executor/wasmtime/src/tests.rs @@ -384,7 +384,9 @@ fn test_max_memory_pages( ) (i32.const -1) ) - (unreachable) + (then + (unreachable) + ) ) (i64.const 0) @@ -421,7 +423,9 @@ fn test_max_memory_pages( ) (i32.const -1) ) - (unreachable) + (then + (unreachable) + ) ) (i64.const 0) diff --git a/substrate/client/mixnet/Cargo.toml b/substrate/client/mixnet/Cargo.toml index 86c5a37754af..d11cb1805ff0 100644 --- a/substrate/client/mixnet/Cargo.toml +++ b/substrate/client/mixnet/Cargo.toml @@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] array-bytes = "4.1" arrayvec = "0.7.2" blake2 = "0.10.4" +bytes = "1" codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] } futures = "0.3.25" futures-timer = "3.0.2" diff --git a/substrate/client/mixnet/src/packet_dispatcher.rs b/substrate/client/mixnet/src/packet_dispatcher.rs index 856208ecb342..420e0c68847d 100644 --- a/substrate/client/mixnet/src/packet_dispatcher.rs +++ b/substrate/client/mixnet/src/packet_dispatcher.rs @@ -24,7 +24,7 @@ use libp2p_identity::PeerId; use log::{debug, warn}; use mixnet::core::{AddressedPacket, NetworkStatus, Packet, PeerId as CorePeerId}; use parking_lot::Mutex; -use sc_network::{NetworkNotification, ProtocolName}; +use sc_network::NotificationService; use std::{collections::HashMap, future::Future, sync::Arc}; const LOG_TARGET: &str = "mixnet"; @@ -77,41 +77,37 @@ pub struct ReadyPeer { } impl ReadyPeer { - /// If a future is returned, and if that future returns `Some`, this function should be called - /// again to send the next packet queued for the peer; `self` is placed in the `Some` to make - /// this straightforward. Otherwise, we have either sent or dropped all packets queued for the - /// peer, and it can be forgotten about for the time being. + /// If a future is returned, and if that future returns `Some`, this function should be + /// called again to send the next packet queued for the peer; `self` is placed in the `Some` + /// to make this straightforward. Otherwise, we have either sent or dropped all packets + /// queued for the peer, and it can be forgotten about for the time being. pub fn send_packet( self, - network: &impl NetworkNotification, - protocol_name: ProtocolName, + notification_service: &Box, ) -> Option>> { - match network.notification_sender(self.id, protocol_name) { - Err(err) => { + match notification_service.message_sink(&self.id) { + None => { debug!( target: LOG_TARGET, - "Failed to get notification sender for peer ID {}: {err}", self.id + "Failed to get message sink for peer ID {}", self.id, ); self.queue.clear(); None }, - Ok(sender) => Some(async move { - match sender.ready().await.and_then(|mut ready| { - let (packet, more_packets) = self.queue.pop(); - let packet = - packet.expect("Should only be called if there is a packet to send"); - ready.send((packet as Box<[_]>).into())?; - Ok(more_packets) - }) { + Some(sink) => Some(async move { + let (packet, more_packets) = self.queue.pop(); + let packet = packet.expect("Should only be called if there is a packet to send"); + + match sink.send_async_notification((packet as Box<[_]>).into()).await { + Ok(_) => more_packets.then_some(self), Err(err) => { debug!( target: LOG_TARGET, - "Notification sender for peer ID {} failed: {err}", self.id + "Failed to send packet to peer ID {}: {err}", self.id, ); self.queue.clear(); None }, - Ok(more_packets) => more_packets.then(|| self), } }), } diff --git a/substrate/client/mixnet/src/protocol.rs b/substrate/client/mixnet/src/protocol.rs index 555c267b86e0..955502a4856a 100644 --- a/substrate/client/mixnet/src/protocol.rs +++ b/substrate/client/mixnet/src/protocol.rs @@ -18,7 +18,10 @@ use super::config::Config; use mixnet::core::PACKET_SIZE; -use sc_network::{config::NonDefaultSetConfig, ProtocolName}; +use sc_network::{ + config::{NonDefaultSetConfig, NonReservedPeerMode, SetConfig}, + NotificationService, ProtocolName, +}; /// Returns the protocol name to use for the mixnet controlled by the given chain. pub fn protocol_name(genesis_hash: &[u8], fork_id: Option<&str>) -> ProtocolName { @@ -31,12 +34,26 @@ pub fn protocol_name(genesis_hash: &[u8], fork_id: Option<&str>) -> ProtocolName } /// Returns the peers set configuration for the mixnet protocol. -pub fn peers_set_config(name: ProtocolName, config: &Config) -> NonDefaultSetConfig { - let mut set_config = NonDefaultSetConfig::new(name, PACKET_SIZE as u64); +pub fn peers_set_config( + name: ProtocolName, + config: &Config, +) -> (NonDefaultSetConfig, Box) { + let (mut set_config, service) = NonDefaultSetConfig::new( + name, + Vec::new(), + PACKET_SIZE as u64, + None, + SetConfig { + in_peers: 0, + out_peers: 0, + reserved_nodes: Vec::new(), + non_reserved_mode: NonReservedPeerMode::Deny, + }, + ); if config.substrate.num_gateway_slots != 0 { // out_peers is always 0; we are only interested in connecting to mixnodes, which we do by // setting them as reserved nodes set_config.allow_non_reserved(config.substrate.num_gateway_slots, 0); } - set_config + (set_config, service) } diff --git a/substrate/client/mixnet/src/run.rs b/substrate/client/mixnet/src/run.rs index 09020469d5ee..14d188df0977 100644 --- a/substrate/client/mixnet/src/run.rs +++ b/substrate/client/mixnet/src/run.rs @@ -29,11 +29,12 @@ use super::{ request::{extrinsic_delay, Request, SUBMIT_EXTRINSIC}, sync_with_runtime::sync_with_runtime, }; +use bytes::Bytes; use codec::{Decode, DecodeAll, Encode}; use futures::{ future::{pending, Either}, stream::FuturesUnordered, - StreamExt, + FutureExt, StreamExt, }; use log::{debug, error, trace, warn}; use mixnet::{ @@ -43,8 +44,8 @@ use mixnet::{ }; use sc_client_api::{BlockchainEvents, HeaderBackend}; use sc_network::{ - Event::{NotificationStreamClosed, NotificationStreamOpened, NotificationsReceived}, - NetworkEventStream, NetworkNotification, NetworkPeers, NetworkStateInfo, ProtocolName, + service::traits::{NotificationEvent, ValidationResult}, + NetworkNotification, NetworkPeers, NetworkStateInfo, NotificationService, ProtocolName, }; use sc_transaction_pool_api::{ LocalTransactionPool, OffchainTransactionPoolFactory, TransactionPool, @@ -154,12 +155,13 @@ pub async fn run( protocol_name: ProtocolName, transaction_pool: Arc

, keystore: Option, + mut notification_service: Box, ) where B: Block, C: BlockchainEvents + ProvideRuntimeApi + HeaderBackend, C::Api: MixnetApi, S: SyncOracle, - N: NetworkStateInfo + NetworkEventStream + NetworkNotification + NetworkPeers, + N: NetworkStateInfo + NetworkNotification + NetworkPeers, P: TransactionPool + LocalTransactionPool + 'static, { let local_peer_id = network.local_peer_id(); @@ -189,7 +191,6 @@ pub async fn run( } else { None }; - let mut network_events = network.event_stream("mixnet").fuse(); let mut next_forward_packet_delay = MaybeInfDelay::new(None); let mut next_authored_packet_delay = MaybeInfDelay::new(None); let mut ready_peers = FuturesUnordered::new(); @@ -248,33 +249,36 @@ pub async fn run( } } - event = network_events.select_next_some() => match event { - NotificationStreamOpened { remote, protocol, .. } - if protocol == protocol_name => packet_dispatcher.add_peer(&remote), - NotificationStreamClosed { remote, protocol } - if protocol == protocol_name => packet_dispatcher.remove_peer(&remote), - NotificationsReceived { remote, messages } => { - for message in messages { - if message.0 == protocol_name { - match message.1.as_ref().try_into() { - Ok(packet) => handle_packet(packet, - &mut mixnet, &mut request_manager, &mut reply_manager, - &mut extrinsic_queue, &config.substrate), - Err(_) => debug!(target: LOG_TARGET, - "Dropped incorrectly sized packet ({} bytes) from {remote}", - message.1.len(), - ), - } - } + event = notification_service.next_event().fuse() => match event { + None => todo!(), + Some(NotificationEvent::ValidateInboundSubstream { result_tx, .. }) => { + let _ = result_tx.send(ValidationResult::Accept); + }, + Some(NotificationEvent::NotificationStreamOpened { peer, .. }) => { + packet_dispatcher.add_peer(&peer); + }, + Some(NotificationEvent::NotificationStreamClosed { peer }) => { + packet_dispatcher.remove_peer(&peer); + }, + Some(NotificationEvent::NotificationReceived { peer, notification }) => { + let notification: Bytes = notification.into(); + + match notification.as_ref().try_into() { + Ok(packet) => handle_packet(packet, + &mut mixnet, &mut request_manager, &mut reply_manager, + &mut extrinsic_queue, &config.substrate), + Err(_) => debug!(target: LOG_TARGET, + "Dropped incorrectly sized packet ({} bytes) from {peer}", + notification.len(), + ), } - } - _ => () + }, }, _ = next_forward_packet_delay => { if let Some(packet) = mixnet.pop_next_forward_packet() { if let Some(ready_peer) = packet_dispatcher.dispatch(packet) { - if let Some(fut) = ready_peer.send_packet(&*network, protocol_name.clone()) { + if let Some(fut) = ready_peer.send_packet(¬ification_service) { ready_peers.push(fut); } } @@ -288,7 +292,7 @@ pub async fn run( _ = next_authored_packet_delay => { if let Some(packet) = mixnet.pop_next_authored_packet(&packet_dispatcher) { if let Some(ready_peer) = packet_dispatcher.dispatch(packet) { - if let Some(fut) = ready_peer.send_packet(&*network, protocol_name.clone()) { + if let Some(fut) = ready_peer.send_packet(¬ification_service) { ready_peers.push(fut); } } @@ -297,7 +301,7 @@ pub async fn run( ready_peer = ready_peers.select_next_some() => { if let Some(ready_peer) = ready_peer { - if let Some(fut) = ready_peer.send_packet(&*network, protocol_name.clone()) { + if let Some(fut) = ready_peer.send_packet(¬ification_service) { ready_peers.push(fut); } } diff --git a/substrate/client/mixnet/src/sync_with_runtime.rs b/substrate/client/mixnet/src/sync_with_runtime.rs index 4a80b3c75f43..f3be96025410 100644 --- a/substrate/client/mixnet/src/sync_with_runtime.rs +++ b/substrate/client/mixnet/src/sync_with_runtime.rs @@ -196,6 +196,7 @@ where #[cfg(test)] mod tests { use super::*; + use multiaddr::multiaddr; #[test] fn fixup_empty_external_addresses() { diff --git a/substrate/client/network-gossip/Cargo.toml b/substrate/client/network-gossip/Cargo.toml index 95e26a232c1d..5006d5d0e3e3 100644 --- a/substrate/client/network-gossip/Cargo.toml +++ b/substrate/client/network-gossip/Cargo.toml @@ -29,5 +29,7 @@ sp-runtime = { path = "../../primitives/runtime" } [dev-dependencies] tokio = "1.22.0" +async-trait = "0.1.73" +codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive" ] } quickcheck = { version = "1.0.3", default-features = false } substrate-test-runtime-client = { path = "../../test-utils/runtime/client" } diff --git a/substrate/client/network-gossip/src/bridge.rs b/substrate/client/network-gossip/src/bridge.rs index 8f7d490757b3..c1bc414c3a34 100644 --- a/substrate/client/network-gossip/src/bridge.rs +++ b/substrate/client/network-gossip/src/bridge.rs @@ -21,7 +21,11 @@ use crate::{ Network, Syncing, Validator, }; -use sc_network::{event::Event, types::ProtocolName, ReputationChange}; +use sc_network::{ + service::traits::{NotificationEvent, ValidationResult}, + types::ProtocolName, + NotificationService, ReputationChange, +}; use sc_network_sync::SyncEvent; use futures::{ @@ -48,10 +52,10 @@ pub struct GossipEngine { periodic_maintenance_interval: futures_timer::Delay, protocol: ProtocolName, - /// Incoming events from the network. - network_event_stream: Pin + Send>>, /// Incoming events from the syncing service. sync_event_stream: Pin + Send>>, + /// Handle for polling notification-related events. + notification_service: Box, /// Outgoing events to the consumer. message_sinks: HashMap>>, /// Buffered messages (see [`ForwardingState`]). @@ -81,6 +85,7 @@ impl GossipEngine { pub fn new( network: N, sync: S, + notification_service: Box, protocol: impl Into, validator: Arc>, metrics_registry: Option<&Registry>, @@ -91,17 +96,16 @@ impl GossipEngine { S: Syncing + Send + Clone + 'static, { let protocol = protocol.into(); - let network_event_stream = network.event_stream("network-gossip"); let sync_event_stream = sync.event_stream("network-gossip"); GossipEngine { state_machine: ConsensusGossip::new(validator, protocol.clone(), metrics_registry), network: Box::new(network), sync: Box::new(sync), + notification_service, periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL), protocol, - network_event_stream, sync_event_stream, message_sinks: HashMap::new(), forwarding_state: ForwardingState::Idle, @@ -125,7 +129,7 @@ impl GossipEngine { /// Broadcast all messages with given topic. pub fn broadcast_topic(&mut self, topic: B::Hash, force: bool) { - self.state_machine.broadcast_topic(&mut *self.network, topic, force); + self.state_machine.broadcast_topic(&mut self.notification_service, topic, force); } /// Get data of valid, incoming messages for a topic (but might have expired meanwhile). @@ -150,19 +154,21 @@ impl GossipEngine { /// Send all messages with given topic to a peer. pub fn send_topic(&mut self, who: &PeerId, topic: B::Hash, force: bool) { - self.state_machine.send_topic(&mut *self.network, who, topic, force) + self.state_machine.send_topic(&mut self.notification_service, who, topic, force) } /// Multicast a message to all peers. pub fn gossip_message(&mut self, topic: B::Hash, message: Vec, force: bool) { - self.state_machine.multicast(&mut *self.network, topic, message, force) + self.state_machine + .multicast(&mut self.notification_service, topic, message, force) } /// Send addressed message to the given peers. The message is not kept or multicast /// later on. pub fn send_message(&mut self, who: Vec, data: Vec) { for who in &who { - self.state_machine.send_message(&mut *self.network, who, data.clone()); + self.state_machine + .send_message(&mut self.notification_service, who, data.clone()); } } @@ -173,6 +179,11 @@ impl GossipEngine { pub fn announce(&self, block: B::Hash, associated_data: Option>) { self.sync.announce_block(block, associated_data); } + + /// Consume [`GossipEngine`] and return the notification service. + pub fn take_notification_service(self) -> Box { + self.notification_service + } } impl Future for GossipEngine { @@ -184,46 +195,56 @@ impl Future for GossipEngine { 'outer: loop { match &mut this.forwarding_state { ForwardingState::Idle => { - let net_event_stream = this.network_event_stream.poll_next_unpin(cx); + let next_notification_event = + this.notification_service.next_event().poll_unpin(cx); let sync_event_stream = this.sync_event_stream.poll_next_unpin(cx); - if net_event_stream.is_pending() && sync_event_stream.is_pending() { + if next_notification_event.is_pending() && sync_event_stream.is_pending() { break } - match net_event_stream { + match next_notification_event { Poll::Ready(Some(event)) => match event { - Event::NotificationStreamOpened { remote, protocol, role, .. } => - if protocol == this.protocol { - this.state_machine.new_peer(&mut *this.network, remote, role); - }, - Event::NotificationStreamClosed { remote, protocol } => { - if protocol == this.protocol { - this.state_machine - .peer_disconnected(&mut *this.network, remote); - } + NotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + .. + } => { + // only accept peers whose role can be determined + let result = this + .network + .peer_role(peer, handshake) + .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); + let _ = result_tx.send(result); }, - Event::NotificationsReceived { remote, messages } => { - let messages = messages - .into_iter() - .filter_map(|(engine, data)| { - if engine == this.protocol { - Some(data.to_vec()) - } else { - None - } - }) - .collect(); - + NotificationEvent::NotificationStreamOpened { + peer, handshake, .. + } => { + let Some(role) = this.network.peer_role(peer, handshake) else { + log::debug!(target: "gossip", "role for {peer} couldn't be determined"); + continue + }; + + this.state_machine.new_peer( + &mut this.notification_service, + peer, + role, + ); + }, + NotificationEvent::NotificationStreamClosed { peer } => { + this.state_machine + .peer_disconnected(&mut this.notification_service, peer); + }, + NotificationEvent::NotificationReceived { peer, notification } => { let to_forward = this.state_machine.on_incoming( &mut *this.network, - remote, - messages, + &mut this.notification_service, + peer, + vec![notification], ); - this.forwarding_state = ForwardingState::Busy(to_forward.into()); }, - Event::Dht(_) => {}, }, // The network event stream closed. Do the same for [`GossipValidator`]. Poll::Ready(None) => { @@ -306,7 +327,7 @@ impl Future for GossipEngine { while let Poll::Ready(()) = this.periodic_maintenance_interval.poll_unpin(cx) { this.periodic_maintenance_interval.reset(PERIODIC_MAINTENANCE_INTERVAL); - this.state_machine.tick(&mut *this.network); + this.state_machine.tick(&mut this.notification_service); this.message_sinks.retain(|_, sinks| { sinks.retain(|sink| !sink.is_closed()); @@ -328,15 +349,19 @@ impl futures::future::FusedFuture for GossipEngine { mod tests { use super::*; use crate::{multiaddr::Multiaddr, ValidationResult, ValidatorContext}; + use codec::{DecodeAll, Encode}; use futures::{ - channel::mpsc::{unbounded, UnboundedSender}, + channel::mpsc::{unbounded, UnboundedReceiver, UnboundedSender}, executor::{block_on, block_on_stream}, future::poll_fn, }; use quickcheck::{Arbitrary, Gen, QuickCheck}; use sc_network::{ - config::MultiaddrWithPeerId, NetworkBlock, NetworkEventStream, NetworkNotification, - NetworkPeers, NotificationSenderError, NotificationSenderT as NotificationSender, + config::MultiaddrWithPeerId, + service::traits::{Direction, MessageSink, NotificationEvent}, + Event, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, + NotificationSenderError, NotificationSenderT as NotificationSender, NotificationService, + Roles, }; use sc_network_common::role::ObservedRole; use sc_network_sync::SyncEventStream; @@ -351,14 +376,10 @@ mod tests { use substrate_test_runtime_client::runtime::Block; #[derive(Clone, Default)] - struct TestNetwork { - inner: Arc>, - } + struct TestNetwork {} #[derive(Clone, Default)] - struct TestNetworkInner { - event_senders: Vec>, - } + struct TestNetworkInner {} impl NetworkPeers for TestNetwork { fn set_authorized_peers(&self, _peers: HashSet) { @@ -422,14 +443,17 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(ObservedRole::from(role))) + } } impl NetworkEventStream for TestNetwork { fn event_stream(&self, _name: &'static str) -> Pin + Send>> { - let (tx, rx) = unbounded(); - self.inner.lock().unwrap().event_senders.push(tx); - - Box::pin(rx) + unimplemented!(); } } @@ -501,6 +525,58 @@ mod tests { } } + #[derive(Debug)] + pub(crate) struct TestNotificationService { + rx: UnboundedReceiver, + } + + #[async_trait::async_trait] + impl sc_network::service::traits::NotificationService for TestNotificationService { + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { + unimplemented!(); + } + + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + fn try_set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + async fn next_event(&mut self) -> Option { + self.rx.next().await + } + + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } + + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } + + fn message_sink(&self, _peer: &PeerId) -> Option> { + unimplemented!(); + } + } + struct AllowAll; impl Validator for AllowAll { fn validate( @@ -521,16 +597,19 @@ mod tests { fn returns_when_network_event_stream_closes() { let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let mut gossip_engine = GossipEngine::::new( network.clone(), sync, + notification_service, "/my_protocol", Arc::new(AllowAll {}), None, ); - // Drop network event stream sender side. - drop(network.inner.lock().unwrap().event_senders.pop()); + // drop notification service sender side. + drop(tx); block_on(poll_fn(move |ctx| { if let Poll::Pending = gossip_engine.poll_unpin(ctx) { @@ -550,42 +629,37 @@ mod tests { let remote_peer = PeerId::random(); let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (mut tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let mut gossip_engine = GossipEngine::::new( network.clone(), sync.clone(), + notification_service, protocol.clone(), Arc::new(AllowAll {}), None, ); - let mut event_sender = network.inner.lock().unwrap().event_senders.pop().unwrap(); - // Register the remote peer. - event_sender - .start_send(Event::NotificationStreamOpened { - remote: remote_peer, - protocol: protocol.clone(), - negotiated_fallback: None, - role: ObservedRole::Authority, - received_handshake: vec![], - }) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationStreamOpened { + peer: remote_peer, + direction: Direction::Inbound, + negotiated_fallback: None, + handshake: Roles::FULL.encode(), + }) + .await + .unwrap(); let messages = vec![vec![1], vec![2]]; - let events = messages - .iter() - .cloned() - .map(|m| Event::NotificationsReceived { - remote: remote_peer, - messages: vec![(protocol.clone(), m.into())], - }) - .collect::>(); // Send first event before subscribing. - event_sender - .start_send(events[0].clone()) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: messages[0].clone().into(), + }) + .await + .unwrap(); let mut subscribers = vec![]; for _ in 0..2 { @@ -593,9 +667,12 @@ mod tests { } // Send second event after subscribing. - event_sender - .start_send(events[1].clone()) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: messages[1].clone().into(), + }) + .await + .unwrap(); tokio::spawn(gossip_engine); @@ -672,6 +749,8 @@ mod tests { let remote_peer = PeerId::random(); let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (mut tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let num_channels_per_topic = channels.iter().fold( HashMap::new(), @@ -699,6 +778,7 @@ mod tests { let mut gossip_engine = GossipEngine::::new( network.clone(), sync.clone(), + notification_service, protocol.clone(), Arc::new(TestValidator {}), None, @@ -724,22 +804,18 @@ mod tests { } } - let mut event_sender = network.inner.lock().unwrap().event_senders.pop().unwrap(); - // Register the remote peer. - event_sender - .start_send(Event::NotificationStreamOpened { - remote: remote_peer, - protocol: protocol.clone(), - negotiated_fallback: None, - role: ObservedRole::Authority, - received_handshake: vec![], - }) - .expect("Event stream is unbounded; qed."); + tx.start_send(NotificationEvent::NotificationStreamOpened { + peer: remote_peer, + direction: Direction::Inbound, + negotiated_fallback: None, + handshake: Roles::FULL.encode(), + }) + .unwrap(); // Send messages into the network event stream. for (i_notification, messages) in notifications.iter().enumerate() { - let messages = messages + let messages: Vec> = messages .into_iter() .enumerate() .map(|(i_message, Message { topic })| { @@ -752,13 +828,17 @@ mod tests { message.push(i_notification.try_into().unwrap()); message.push(i_message.try_into().unwrap()); - (protocol.clone(), message.into()) + message.into() }) .collect(); - event_sender - .start_send(Event::NotificationsReceived { remote: remote_peer, messages }) - .expect("Event stream is unbounded; qed."); + for message in messages { + tx.start_send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: message, + }) + .unwrap(); + } } let mut received_msgs_per_topic_all_chan = HashMap::::new(); diff --git a/substrate/client/network-gossip/src/state_machine.rs b/substrate/client/network-gossip/src/state_machine.rs index 4bfb5a7d37f4..91b56b0f097e 100644 --- a/substrate/client/network-gossip/src/state_machine.rs +++ b/substrate/client/network-gossip/src/state_machine.rs @@ -23,7 +23,7 @@ use libp2p::PeerId; use schnellru::{ByLength, LruMap}; use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; -use sc_network::types::ProtocolName; +use sc_network::{types::ProtocolName, NotificationService}; use sc_network_common::role::ObservedRole; use sp_runtime::traits::{Block as BlockT, Hash, HashingFor}; use std::{collections::HashMap, iter, sync::Arc, time, time::Instant}; @@ -74,33 +74,33 @@ struct MessageEntry { /// Local implementation of `ValidatorContext`. struct NetworkContext<'g, 'p, B: BlockT> { gossip: &'g mut ConsensusGossip, - network: &'p mut dyn Network, + notification_service: &'p mut Box, } impl<'g, 'p, B: BlockT> ValidatorContext for NetworkContext<'g, 'p, B> { /// Broadcast all messages with given topic to peers that do not have it yet. fn broadcast_topic(&mut self, topic: B::Hash, force: bool) { - self.gossip.broadcast_topic(self.network, topic, force); + self.gossip.broadcast_topic(self.notification_service, topic, force); } /// Broadcast a message to all peers that have not received it previously. fn broadcast_message(&mut self, topic: B::Hash, message: Vec, force: bool) { - self.gossip.multicast(self.network, topic, message, force); + self.gossip.multicast(self.notification_service, topic, message, force); } /// Send addressed message to a peer. fn send_message(&mut self, who: &PeerId, message: Vec) { - self.network.write_notification(*who, self.gossip.protocol.clone(), message); + self.notification_service.send_sync_notification(who, message); } /// Send all messages with given topic to a peer. fn send_topic(&mut self, who: &PeerId, topic: B::Hash, force: bool) { - self.gossip.send_topic(self.network, who, topic, force); + self.gossip.send_topic(self.notification_service, who, topic, force); } } fn propagate<'a, B: BlockT, I>( - network: &mut dyn Network, + notification_service: &mut Box, protocol: ProtocolName, messages: I, intent: MessageIntent, @@ -147,7 +147,7 @@ where ?message, "Propagating message", ); - network.write_notification(*id, protocol.clone(), message.clone()); + notification_service.send_sync_notification(id, message.clone()); } } } @@ -191,7 +191,12 @@ impl ConsensusGossip { } /// Handle new connected peer. - pub fn new_peer(&mut self, network: &mut dyn Network, who: PeerId, role: ObservedRole) { + pub fn new_peer( + &mut self, + notification_service: &mut Box, + who: PeerId, + role: ObservedRole, + ) { tracing::trace!( target:"gossip", %who, @@ -202,7 +207,7 @@ impl ConsensusGossip { self.peers.insert(who, PeerConsensus { known_messages: Default::default() }); let validator = self.validator.clone(); - let mut context = NetworkContext { gossip: self, network }; + let mut context = NetworkContext { gossip: self, notification_service }; validator.new_peer(&mut context, &who, role); } @@ -233,30 +238,35 @@ impl ConsensusGossip { } /// Call when a peer has been disconnected to stop tracking gossip status. - pub fn peer_disconnected(&mut self, network: &mut dyn Network, who: PeerId) { + pub fn peer_disconnected( + &mut self, + notification_service: &mut Box, + who: PeerId, + ) { let validator = self.validator.clone(); - let mut context = NetworkContext { gossip: self, network }; + let mut context = NetworkContext { gossip: self, notification_service }; validator.peer_disconnected(&mut context, &who); self.peers.remove(&who); } /// Perform periodic maintenance - pub fn tick(&mut self, network: &mut dyn Network) { + pub fn tick(&mut self, notification_service: &mut Box) { self.collect_garbage(); if Instant::now() >= self.next_broadcast { - self.rebroadcast(network); + self.rebroadcast(notification_service); self.next_broadcast = Instant::now() + REBROADCAST_INTERVAL; } } /// Rebroadcast all messages to all peers. - fn rebroadcast(&mut self, network: &mut dyn Network) { + fn rebroadcast(&mut self, notification_service: &mut Box) { let messages = self .messages .iter() .map(|entry| (&entry.message_hash, &entry.topic, &entry.message)); + propagate( - network, + notification_service, self.protocol.clone(), messages, MessageIntent::PeriodicRebroadcast, @@ -266,7 +276,12 @@ impl ConsensusGossip { } /// Broadcast all messages with given topic. - pub fn broadcast_topic(&mut self, network: &mut dyn Network, topic: B::Hash, force: bool) { + pub fn broadcast_topic( + &mut self, + notification_service: &mut Box, + topic: B::Hash, + force: bool, + ) { let messages = self.messages.iter().filter_map(|entry| { if entry.topic == topic { Some((&entry.message_hash, &entry.topic, &entry.message)) @@ -276,7 +291,7 @@ impl ConsensusGossip { }); let intent = if force { MessageIntent::ForcedBroadcast } else { MessageIntent::Broadcast }; propagate( - network, + notification_service, self.protocol.clone(), messages, intent, @@ -327,6 +342,7 @@ impl ConsensusGossip { pub fn on_incoming( &mut self, network: &mut dyn Network, + notification_service: &mut Box, who: PeerId, messages: Vec>, ) -> Vec<(B::Hash, TopicNotification)> { @@ -367,7 +383,7 @@ impl ConsensusGossip { // validate the message let validation = { let validator = self.validator.clone(); - let mut context = NetworkContext { gossip: self, network }; + let mut context = NetworkContext { gossip: self, notification_service }; validator.validate(&mut context, &who, &message) }; @@ -414,7 +430,7 @@ impl ConsensusGossip { /// Send all messages with given topic to a peer. pub fn send_topic( &mut self, - network: &mut dyn Network, + notification_service: &mut Box, who: &PeerId, topic: B::Hash, force: bool, @@ -443,7 +459,7 @@ impl ConsensusGossip { ?entry.message, "Sending topic message", ); - network.write_notification(*who, self.protocol.clone(), entry.message.clone()); + notification_service.send_sync_notification(who, entry.message.clone()); } } } @@ -451,7 +467,7 @@ impl ConsensusGossip { /// Multicast a message to all peers. pub fn multicast( &mut self, - network: &mut dyn Network, + notification_service: &mut Box, topic: B::Hash, message: Vec, force: bool, @@ -460,7 +476,7 @@ impl ConsensusGossip { self.register_message_hashed(message_hash, topic, message.clone(), None); let intent = if force { MessageIntent::ForcedBroadcast } else { MessageIntent::Broadcast }; propagate( - network, + notification_service, self.protocol.clone(), iter::once((&message_hash, &topic, &message)), intent, @@ -471,7 +487,12 @@ impl ConsensusGossip { /// Send addressed message to a peer. The message is not kept or multicast /// later on. - pub fn send_message(&mut self, network: &mut dyn Network, who: &PeerId, message: Vec) { + pub fn send_message( + &mut self, + notification_service: &mut Box, + who: &PeerId, + message: Vec, + ) { let peer = match self.peers.get_mut(who) { None => return, Some(peer) => peer, @@ -488,7 +509,7 @@ impl ConsensusGossip { ); peer.known_messages.insert(message_hash); - network.write_notification(*who, self.protocol.clone(), message); + notification_service.send_sync_notification(who, message) } } @@ -524,9 +545,9 @@ mod tests { use crate::multiaddr::Multiaddr; use futures::prelude::*; use sc_network::{ - config::MultiaddrWithPeerId, event::Event, NetworkBlock, NetworkEventStream, - NetworkNotification, NetworkPeers, NotificationSenderError, - NotificationSenderT as NotificationSender, ReputationChange, + config::MultiaddrWithPeerId, event::Event, service::traits::NotificationEvent, MessageSink, + NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, + NotificationSenderError, NotificationSenderT as NotificationSender, ReputationChange, }; use sp_runtime::{ testing::{Block as RawBlock, ExtrinsicWrapper, H256}, @@ -651,6 +672,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } impl NetworkEventStream for NoOpNetwork { @@ -691,6 +716,62 @@ mod tests { } } + #[derive(Debug, Default)] + struct NoOpNotificationService {} + + #[async_trait::async_trait] + impl NotificationService for NoOpNotificationService { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { + unimplemented!(); + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + fn try_set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + None + } + + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } + + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } + + fn message_sink(&self, _peer: &PeerId) -> Option> { + unimplemented!(); + } + } + #[test] fn collects_garbage() { struct AllowOne; @@ -773,20 +854,28 @@ mod tests { fn peer_is_removed_on_disconnect() { let mut consensus = ConsensusGossip::::new(Arc::new(AllowAll), "/foo".into(), None); - let mut network = NoOpNetwork::default(); + let mut notification_service: Box = + Box::new(NoOpNotificationService::default()); let peer_id = PeerId::random(); - consensus.new_peer(&mut network, peer_id, ObservedRole::Full); + consensus.new_peer(&mut notification_service, peer_id, ObservedRole::Full); assert!(consensus.peers.contains_key(&peer_id)); - consensus.peer_disconnected(&mut network, peer_id); + consensus.peer_disconnected(&mut notification_service, peer_id); assert!(!consensus.peers.contains_key(&peer_id)); } #[test] fn on_incoming_ignores_discarded_messages() { + let mut notification_service: Box = + Box::new(NoOpNotificationService::default()); let to_forward = ConsensusGossip::::new(Arc::new(DiscardAll), "/foo".into(), None) - .on_incoming(&mut NoOpNetwork::default(), PeerId::random(), vec![vec![1, 2, 3]]); + .on_incoming( + &mut NoOpNetwork::default(), + &mut notification_service, + PeerId::random(), + vec![vec![1, 2, 3]], + ); assert!( to_forward.is_empty(), @@ -798,11 +887,14 @@ mod tests { #[test] fn on_incoming_ignores_unregistered_peer() { let mut network = NoOpNetwork::default(); + let mut notification_service: Box = + Box::new(NoOpNotificationService::default()); let remote = PeerId::random(); let to_forward = ConsensusGossip::::new(Arc::new(AllowAll), "/foo".into(), None) .on_incoming( &mut network, + &mut notification_service, // Unregistered peer. remote, vec![vec![1, 2, 3]], @@ -822,18 +914,20 @@ mod tests { let mut consensus = ConsensusGossip::::new(Arc::new(AllowAll), "/foo".into(), None); let mut network = NoOpNetwork::default(); + let mut notification_service: Box = + Box::new(NoOpNotificationService::default()); let peer_id = PeerId::random(); - consensus.new_peer(&mut network, peer_id, ObservedRole::Full); + consensus.new_peer(&mut notification_service, peer_id, ObservedRole::Full); assert!(consensus.peers.contains_key(&peer_id)); let peer_id2 = PeerId::random(); - consensus.new_peer(&mut network, peer_id2, ObservedRole::Full); + consensus.new_peer(&mut notification_service, peer_id2, ObservedRole::Full); assert!(consensus.peers.contains_key(&peer_id2)); let message = vec![vec![1, 2, 3]]; - consensus.on_incoming(&mut network, peer_id, message.clone()); - consensus.on_incoming(&mut network, peer_id2, message.clone()); + consensus.on_incoming(&mut network, &mut notification_service, peer_id, message.clone()); + consensus.on_incoming(&mut network, &mut notification_service, peer_id2, message.clone()); assert_eq!( vec![(peer_id, rep::GOSSIP_SUCCESS)], diff --git a/substrate/client/network/Cargo.toml b/substrate/client/network/Cargo.toml index 7b0536addda4..8b599f058f76 100644 --- a/substrate/client/network/Cargo.toml +++ b/substrate/client/network/Cargo.toml @@ -37,6 +37,8 @@ serde = { version = "1.0.188", features = ["derive"] } serde_json = "1.0.108" smallvec = "1.11.0" thiserror = "1.0" +tokio = { version = "1.22.0", features = ["macros", "sync"] } +tokio-stream = "0.1.7" unsigned-varint = { version = "0.7.1", features = ["futures", "asynchronous_codec"] } zeroize = "1.4.3" prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" } diff --git a/substrate/client/network/common/src/role.rs b/substrate/client/network/common/src/role.rs index fd02c00e2324..11b7a7924c46 100644 --- a/substrate/client/network/common/src/role.rs +++ b/substrate/client/network/common/src/role.rs @@ -28,7 +28,7 @@ use codec::{self, Encode, EncodeLike, Input, Output}; /// > **Note**: This enum is different from the `Role` enum. The `Role` enum indicates what a /// > node says about itself, while `ObservedRole` is a `Role` merged with the /// > information known locally about that node. -#[derive(Debug, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum ObservedRole { /// Full node. Full, @@ -45,6 +45,18 @@ impl ObservedRole { } } +impl From for ObservedRole { + fn from(roles: Roles) -> Self { + if roles.is_authority() { + ObservedRole::Authority + } else if roles.is_full() { + ObservedRole::Full + } else { + ObservedRole::Light + } + } +} + /// Role of the local node. #[derive(Debug, Clone)] pub enum Role { diff --git a/substrate/client/network/src/behaviour.rs b/substrate/client/network/src/behaviour.rs index 0aa724818e02..9f770bc3ba74 100644 --- a/substrate/client/network/src/behaviour.rs +++ b/substrate/client/network/src/behaviour.rs @@ -22,12 +22,13 @@ use crate::{ peer_info, peer_store::PeerStoreHandle, protocol::{CustomMessageOutcome, NotificationsSink, Protocol}, + protocol_controller::SetId, request_responses::{self, IfDisconnected, ProtocolConfig, RequestFailure}, + service::traits::Direction, types::ProtocolName, ReputationChange, }; -use bytes::Bytes; use futures::channel::oneshot; use libp2p::{ core::Multiaddr, identify::Info as IdentifyInfo, identity::PublicKey, kad::RecordKey, @@ -35,7 +36,6 @@ use libp2p::{ }; use parking_lot::Mutex; -use sc_network_common::role::{ObservedRole, Roles}; use sp_runtime::traits::Block as BlockT; use std::{collections::HashSet, sync::Arc, time::Duration}; @@ -97,8 +97,10 @@ pub enum BehaviourOut { NotificationStreamOpened { /// Node we opened the substream with. remote: PeerId, - /// The concerned protocol. Each protocol uses a different substream. - protocol: ProtocolName, + /// Set ID. + set_id: SetId, + /// Direction of the stream. + direction: Direction, /// If the negotiation didn't use the main name of the protocol (the one in /// `notifications_protocol`), then this field contains which name has actually been /// used. @@ -106,8 +108,6 @@ pub enum BehaviourOut { negotiated_fallback: Option, /// Object that permits sending notifications to the peer. notifications_sink: NotificationsSink, - /// Role of the remote. - role: ObservedRole, /// Received handshake. received_handshake: Vec, }, @@ -120,8 +120,8 @@ pub enum BehaviourOut { NotificationStreamReplaced { /// Id of the peer we are connected to. remote: PeerId, - /// The concerned protocol. Each protocol uses a different substream. - protocol: ProtocolName, + /// Set ID. + set_id: SetId, /// Replacement for the previous [`NotificationsSink`]. notifications_sink: NotificationsSink, }, @@ -131,16 +131,18 @@ pub enum BehaviourOut { NotificationStreamClosed { /// Node we closed the substream with. remote: PeerId, - /// The concerned protocol. Each protocol uses a different substream. - protocol: ProtocolName, + /// Set ID. + set_id: SetId, }, /// Received one or more messages from the given node using the given protocol. NotificationsReceived { /// Node we received the message from. remote: PeerId, + /// Set ID. + set_id: SetId, /// Concerned protocol and associated message. - messages: Vec<(ProtocolName, Bytes)>, + notification: Vec, }, /// We have obtained identity information from a peer, including the addresses it is listening @@ -272,44 +274,33 @@ impl Behaviour { } } -fn reported_roles_to_observed_role(roles: Roles) -> ObservedRole { - if roles.is_authority() { - ObservedRole::Authority - } else if roles.is_full() { - ObservedRole::Full - } else { - ObservedRole::Light - } -} - impl From for BehaviourOut { fn from(event: CustomMessageOutcome) -> Self { match event { CustomMessageOutcome::NotificationStreamOpened { remote, - protocol, + set_id, + direction, negotiated_fallback, - roles, received_handshake, notifications_sink, } => BehaviourOut::NotificationStreamOpened { remote, - protocol, + set_id, + direction, negotiated_fallback, - role: reported_roles_to_observed_role(roles), received_handshake, notifications_sink, }, CustomMessageOutcome::NotificationStreamReplaced { remote, - protocol, + set_id, notifications_sink, - } => BehaviourOut::NotificationStreamReplaced { remote, protocol, notifications_sink }, - CustomMessageOutcome::NotificationStreamClosed { remote, protocol } => - BehaviourOut::NotificationStreamClosed { remote, protocol }, - CustomMessageOutcome::NotificationsReceived { remote, messages } => - BehaviourOut::NotificationsReceived { remote, messages }, - CustomMessageOutcome::None => BehaviourOut::None, + } => BehaviourOut::NotificationStreamReplaced { remote, set_id, notifications_sink }, + CustomMessageOutcome::NotificationStreamClosed { remote, set_id } => + BehaviourOut::NotificationStreamClosed { remote, set_id }, + CustomMessageOutcome::NotificationsReceived { remote, set_id, notification } => + BehaviourOut::NotificationsReceived { remote, set_id, notification }, } } } diff --git a/substrate/client/network/src/config.rs b/substrate/client/network/src/config.rs index 124d73a74dbc..24e96843c32d 100644 --- a/substrate/client/network/src/config.rs +++ b/substrate/client/network/src/config.rs @@ -23,10 +23,11 @@ pub use crate::{ discovery::DEFAULT_KADEMLIA_REPLICATION_FACTOR, - protocol::NotificationsSink, + protocol::{notification_service, NotificationsSink, ProtocolHandlePair}, request_responses::{ IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig, }, + service::traits::NotificationService, types::ProtocolName, }; @@ -47,7 +48,6 @@ pub use sc_network_common::{ ExHashT, }; -use sc_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::Block as BlockT; use std::{ @@ -454,14 +454,14 @@ impl Default for SetConfig { /// /// > **Note**: As new fields might be added in the future, please consider using the `new` method /// > and modifiers instead of creating this struct manually. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct NonDefaultSetConfig { /// Name of the notifications protocols of this set. A substream on this set will be /// considered established once this protocol is open. /// /// > **Note**: This field isn't present for the default set, as this is handled internally /// > by the networking code. - pub notifications_protocol: ProtocolName, + protocol_name: ProtocolName, /// If the remote reports that it doesn't support the protocol indicated in the /// `notifications_protocol` field, then each of these fallback names will be tried one by @@ -469,37 +469,84 @@ pub struct NonDefaultSetConfig { /// /// If a fallback is used, it will be reported in /// `sc_network::protocol::event::Event::NotificationStreamOpened::negotiated_fallback` - pub fallback_names: Vec, + fallback_names: Vec, /// Handshake of the protocol /// /// NOTE: Currently custom handshakes are not fully supported. See issue #5685 for more /// details. This field is temporarily used to allow moving the hardcoded block announcement /// protocol out of `protocol.rs`. - pub handshake: Option, + handshake: Option, /// Maximum allowed size of single notifications. - pub max_notification_size: u64, + max_notification_size: u64, /// Base configuration. - pub set_config: SetConfig, + set_config: SetConfig, + + /// Notification handle. + /// + /// Notification handle is created during `NonDefaultSetConfig` creation and its other half, + /// `Box` is given to the protocol created the config and + /// `ProtocolHandle` is given to `Notifications` when it initializes itself. This handle allows + /// `Notifications ` to communicate with the protocol directly without relaying events through + /// `sc-network.` + protocol_handle_pair: ProtocolHandlePair, } impl NonDefaultSetConfig { /// Creates a new [`NonDefaultSetConfig`]. Zero slots and accepts only reserved nodes. - pub fn new(notifications_protocol: ProtocolName, max_notification_size: u64) -> Self { - Self { - notifications_protocol, - max_notification_size, - fallback_names: Vec::new(), - handshake: None, - set_config: SetConfig { - in_peers: 0, - out_peers: 0, - reserved_nodes: Vec::new(), - non_reserved_mode: NonReservedPeerMode::Deny, + /// Also returns an object which allows the protocol to communicate with `Notifications`. + pub fn new( + protocol_name: ProtocolName, + fallback_names: Vec, + max_notification_size: u64, + handshake: Option, + set_config: SetConfig, + ) -> (Self, Box) { + let (protocol_handle_pair, notification_service) = + notification_service(protocol_name.clone()); + ( + Self { + protocol_name, + max_notification_size, + fallback_names, + handshake, + set_config, + protocol_handle_pair, }, - } + notification_service, + ) + } + + /// Get reference to protocol name. + pub fn protocol_name(&self) -> &ProtocolName { + &self.protocol_name + } + + /// Get reference to fallback protocol names. + pub fn fallback_names(&self) -> impl Iterator { + self.fallback_names.iter() + } + + /// Get reference to handshake. + pub fn handshake(&self) -> &Option { + &self.handshake + } + + /// Get maximum notification size. + pub fn max_notification_size(&self) -> u64 { + self.max_notification_size + } + + /// Get reference to `SetConfig`. + pub fn set_config(&self) -> &SetConfig { + &self.set_config + } + + /// Take `ProtocolHandlePair` from `NonDefaultSetConfig` + pub fn take_protocol_handle(self) -> ProtocolHandlePair { + self.protocol_handle_pair } /// Modifies the configuration to allow non-reserved nodes. @@ -703,9 +750,6 @@ pub struct Params { /// Block announce protocol configuration pub block_announce_config: NonDefaultSetConfig, - - /// TX channel for direct communication with `SyncingEngine` and `Protocol`. - pub tx: TracingUnboundedSender>, } /// Full network configuration. diff --git a/substrate/client/network/src/error.rs b/substrate/client/network/src/error.rs index f0828fb821f3..01e8356fb553 100644 --- a/substrate/client/network/src/error.rs +++ b/substrate/client/network/src/error.rs @@ -68,6 +68,15 @@ pub enum Error { /// Name of the protocol registered multiple times. protocol: ProtocolName, }, + /// Peer does not exist. + #[error("Peer `{0}` does not exist.")] + PeerDoesntExist(PeerId), + /// Channel closed. + #[error("Channel closed")] + ChannelClosed, + /// Connection closed. + #[error("Connection closed")] + ConnectionClosed, } // Make `Debug` use the `Display` implementation. diff --git a/substrate/client/network/src/event.rs b/substrate/client/network/src/event.rs index 2913f0b55225..dc4fd53a49aa 100644 --- a/substrate/client/network/src/event.rs +++ b/substrate/client/network/src/event.rs @@ -19,14 +19,12 @@ //! Network event types. These are are not the part of the protocol, but rather //! events that happen on the network like DHT get/put results received. -use crate::{types::ProtocolName, NotificationsSink}; +use crate::types::ProtocolName; use bytes::Bytes; -use futures::channel::oneshot; use libp2p::{kad::record::Key, PeerId}; -use sc_network_common::{role::ObservedRole, sync::message::BlockAnnouncesHandshake}; -use sp_runtime::traits::Block as BlockT; +use sc_network_common::role::ObservedRole; /// Events generated by DHT as a response to get_value and put_value requests. #[derive(Debug, Clone)] @@ -92,46 +90,3 @@ pub enum Event { messages: Vec<(ProtocolName, Bytes)>, }, } - -/// Event sent to `SyncingEngine` -// TODO: remove once `NotificationService` is implemented. -pub enum SyncEvent { - /// Opened a substream with the given node with the given notifications protocol. - /// - /// The protocol is always one of the notification protocols that have been registered. - NotificationStreamOpened { - /// Node we opened the substream with. - remote: PeerId, - /// Received handshake. - received_handshake: BlockAnnouncesHandshake, - /// Notification sink. - sink: NotificationsSink, - /// Is the connection inbound. - inbound: bool, - /// Channel for reporting accept/reject of the substream. - tx: oneshot::Sender, - }, - - /// Closed a substream with the given node. Always matches a corresponding previous - /// `NotificationStreamOpened` message. - NotificationStreamClosed { - /// Node we closed the substream with. - remote: PeerId, - }, - - /// Notification sink was replaced. - NotificationSinkReplaced { - /// Node we closed the substream with. - remote: PeerId, - /// Notification sink. - sink: NotificationsSink, - }, - - /// Received one or more messages from the given node using the given protocol. - NotificationsReceived { - /// Node we received the message from. - remote: PeerId, - /// Concerned protocol and associated message. - messages: Vec, - }, -} diff --git a/substrate/client/network/src/lib.rs b/substrate/client/network/src/lib.rs index 4dc9bdb4cc1c..4c39c57e8dfc 100644 --- a/substrate/client/network/src/lib.rs +++ b/substrate/client/network/src/lib.rs @@ -244,7 +244,6 @@ mod behaviour; mod protocol; -mod service; #[cfg(test)] mod mock; @@ -258,25 +257,30 @@ pub mod peer_info; pub mod peer_store; pub mod protocol_controller; pub mod request_responses; +pub mod service; pub mod transport; pub mod types; pub mod utils; -pub use event::{DhtEvent, Event, SyncEvent}; +pub use event::{DhtEvent, Event}; #[doc(inline)] pub use libp2p::{multiaddr, Multiaddr, PeerId}; pub use request_responses::{Config, IfDisconnected, RequestFailure}; -pub use sc_network_common::{role::ObservedRole, types::ReputationChange}; +pub use sc_network_common::{ + role::{ObservedRole, Roles}, + types::ReputationChange, +}; pub use service::{ signature::Signature, traits::{ - KademliaKey, NetworkBlock, NetworkDHTProvider, NetworkEventStream, NetworkNotification, - NetworkPeers, NetworkRequest, NetworkSigner, NetworkStateInfo, NetworkStatus, - NetworkStatusProvider, NetworkSyncForkRequest, NotificationSender as NotificationSenderT, - NotificationSenderError, NotificationSenderReady, + KademliaKey, MessageSink, NetworkBlock, NetworkDHTProvider, NetworkEventStream, + NetworkNotification, NetworkPeers, NetworkRequest, NetworkSigner, NetworkStateInfo, + NetworkStatus, NetworkStatusProvider, NetworkSyncForkRequest, + NotificationSender as NotificationSenderT, NotificationSenderError, + NotificationSenderReady, NotificationService, }, - DecodingError, Keypair, NetworkService, NetworkWorker, NotificationSender, NotificationsSink, - OutboundFailure, PublicKey, + DecodingError, Keypair, NetworkService, NetworkWorker, NotificationSender, OutboundFailure, + PublicKey, }; pub use types::ProtocolName; diff --git a/substrate/client/network/src/mock.rs b/substrate/client/network/src/mock.rs index bc596b0fa579..534b81189707 100644 --- a/substrate/client/network/src/mock.rs +++ b/substrate/client/network/src/mock.rs @@ -20,6 +20,7 @@ use crate::{peer_store::PeerStoreProvider, protocol_controller::ProtocolHandle, ReputationChange}; use libp2p::PeerId; +use sc_network_common::role::ObservedRole; use std::collections::HashSet; /// No-op `PeerStore`. @@ -49,6 +50,14 @@ impl PeerStoreProvider for MockPeerStore { 0 } + fn peer_role(&self, _peer_id: &PeerId) -> Option { + None + } + + fn set_peer_role(&mut self, _peer_id: &PeerId, _role: ObservedRole) { + unimplemented!(); + } + fn outgoing_candidates(&self, _count: usize, _ignored: HashSet<&PeerId>) -> Vec { unimplemented!() } diff --git a/substrate/client/network/src/peer_store.rs b/substrate/client/network/src/peer_store.rs index 35d17e588cb9..4b28b8e75440 100644 --- a/substrate/client/network/src/peer_store.rs +++ b/substrate/client/network/src/peer_store.rs @@ -23,7 +23,7 @@ use libp2p::PeerId; use log::trace; use parking_lot::Mutex; use partial_sort::PartialSort; -use sc_network_common::types::ReputationChange; +use sc_network_common::{role::ObservedRole, types::ReputationChange}; use std::{ cmp::{Ord, Ordering, PartialOrd}, collections::{hash_map::Entry, HashMap, HashSet}, @@ -66,9 +66,15 @@ pub trait PeerStoreProvider: Debug + Send { /// Adjust peer reputation. fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); + /// Set peer role. + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole); + /// Get peer reputation. fn peer_reputation(&self, peer_id: &PeerId) -> i32; + /// Get peer role, if available. + fn peer_role(&self, peer_id: &PeerId) -> Option; + /// Get candidates with highest reputations for initiating outgoing connections. fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; } @@ -96,10 +102,18 @@ impl PeerStoreProvider for PeerStoreHandle { self.inner.lock().report_peer(peer_id, change) } + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole) { + self.inner.lock().set_peer_role(peer_id, role) + } + fn peer_reputation(&self, peer_id: &PeerId) -> i32 { self.inner.lock().peer_reputation(peer_id) } + fn peer_role(&self, peer_id: &PeerId) -> Option { + self.inner.lock().peer_role(peer_id) + } + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { self.inner.lock().outgoing_candidates(count, ignored) } @@ -122,13 +136,19 @@ impl PeerStoreHandle { #[derive(Debug, Clone, Copy)] struct PeerInfo { + /// Reputation of the peer. reputation: i32, + + /// Instant when the peer was last updated. last_updated: Instant, + + /// Role of the peer, if known. + role: Option, } impl Default for PeerInfo { fn default() -> Self { - Self { reputation: 0, last_updated: Instant::now() } + Self { reputation: 0, last_updated: Instant::now(), role: None } } } @@ -242,10 +262,27 @@ impl PeerStoreInner { } } + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole) { + log::trace!(target: LOG_TARGET, "Set {peer_id} role to {role:?}"); + + match self.peers.entry(*peer_id) { + Entry::Occupied(mut entry) => { + entry.get_mut().role = Some(role); + }, + Entry::Vacant(entry) => { + entry.insert(PeerInfo { role: Some(role), ..Default::default() }); + }, + } + } + fn peer_reputation(&self, peer_id: &PeerId) -> i32 { self.peers.get(peer_id).map_or(0, |info| info.reputation) } + fn peer_role(&self, peer_id: &PeerId) -> Option { + self.peers.get(peer_id).map_or(None, |info| info.role) + } + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { let mut candidates = self .peers diff --git a/substrate/client/network/src/protocol.rs b/substrate/client/network/src/protocol.rs index 9b94f2883528..ea7977cc9ae2 100644 --- a/substrate/client/network/src/protocol.rs +++ b/substrate/client/network/src/protocol.rs @@ -20,12 +20,11 @@ use crate::{ config, error, peer_store::{PeerStoreHandle, PeerStoreProvider}, protocol_controller::{self, SetId}, + service::traits::Direction, types::ProtocolName, }; -use bytes::Bytes; -use codec::{DecodeAll, Encode}; -use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; +use codec::Encode; use libp2p::{ core::Endpoint, swarm::{ @@ -34,24 +33,23 @@ use libp2p::{ }, Multiaddr, PeerId, }; -use log::{debug, error, warn}; +use log::warn; -use sc_network_common::{role::Roles, sync::message::BlockAnnouncesHandshake}; -use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender}; +use codec::DecodeAll; +use prometheus_endpoint::Registry; +use sc_network_common::role::Roles; +use sc_utils::mpsc::TracingUnboundedReceiver; use sp_runtime::traits::Block as BlockT; -use std::{ - collections::{HashMap, HashSet}, - future::Future, - iter, - pin::Pin, - task::Poll, -}; +use std::{collections::HashSet, iter, task::Poll}; -use message::{generic::Message as GenericMessage, Message}; use notifications::{Notifications, NotificationsOut}; -pub use notifications::{NotificationsSink, NotifsHandlerError, Ready}; +pub(crate) use notifications::ProtocolHandle; + +pub use notifications::{ + notification_service, NotificationsSink, NotifsHandlerError, ProtocolHandlePair, Ready, +}; mod notifications; @@ -64,85 +62,93 @@ pub(crate) const BLOCK_ANNOUNCES_TRANSACTIONS_SUBSTREAM_SIZE: u64 = 16 * 1024 * /// Identifier of the peerset for the block announces protocol. const HARDCODED_PEERSETS_SYNC: SetId = SetId::from(0); -mod rep { - use crate::ReputationChange as Rep; - /// We received a message that failed to decode. - pub const BAD_MESSAGE: Rep = Rep::new(-(1 << 12), "Bad message"); -} - -type PendingSyncSubstreamValidation = - Pin> + Send>>; - // Lock must always be taken in order declared here. pub struct Protocol { - /// Used to report reputation changes. - peer_store_handle: PeerStoreHandle, /// Handles opening the unique substream and sending and receiving raw messages. behaviour: Notifications, /// List of notifications protocols that have been registered. notification_protocols: Vec, - /// If we receive a new "substream open" event that contains an invalid handshake, we ask the - /// inner layer to force-close the substream. Force-closing the substream will generate a - /// "substream closed" event. This is a problem: since we can't propagate the "substream open" - /// event to the outer layers, we also shouldn't propagate this "substream closed" event. To - /// solve this, an entry is added to this map whenever an invalid handshake is received. - /// Entries are removed when the corresponding "substream closed" is later received. - bad_handshake_substreams: HashSet<(PeerId, SetId)>, - /// Connected peers on sync protocol. - peers: HashMap, - sync_substream_validations: FuturesUnordered, - tx: TracingUnboundedSender>, + /// Handle to `PeerStore`. + peer_store_handle: PeerStoreHandle, + /// Streams for peers whose handshake couldn't be determined. + bad_handshake_streams: HashSet, + sync_handle: ProtocolHandle, _marker: std::marker::PhantomData, } impl Protocol { /// Create a new instance. - pub fn new( + pub(crate) fn new( roles: Roles, + registry: &Option, notification_protocols: Vec, block_announces_protocol: config::NonDefaultSetConfig, peer_store_handle: PeerStoreHandle, protocol_controller_handles: Vec, from_protocol_controllers: TracingUnboundedReceiver, - tx: TracingUnboundedSender>, - ) -> error::Result { - let behaviour = { - Notifications::new( - protocol_controller_handles, - from_protocol_controllers, - // NOTE: Block announcement protocol is still very much hardcoded into `Protocol`. - // This protocol must be the first notification protocol given to - // `Notifications` - iter::once(notifications::ProtocolConfig { - name: block_announces_protocol.notifications_protocol.clone(), - fallback_names: block_announces_protocol.fallback_names.clone(), - handshake: block_announces_protocol.handshake.as_ref().unwrap().to_vec(), - max_notification_size: block_announces_protocol.max_notification_size, - }) - .chain(notification_protocols.iter().map(|s| notifications::ProtocolConfig { - name: s.notifications_protocol.clone(), - fallback_names: s.fallback_names.clone(), - handshake: s.handshake.as_ref().map_or(roles.encode(), |h| (*h).to_vec()), - max_notification_size: s.max_notification_size, - })), + ) -> error::Result<(Self, Vec)> { + let (behaviour, notification_protocols, handles) = { + let installed_protocols = iter::once(block_announces_protocol.protocol_name().clone()) + .chain(notification_protocols.iter().map(|p| p.protocol_name().clone())) + .collect::>(); + + // NOTE: Block announcement protocol is still very much hardcoded into + // `Protocol`. This protocol must be the first notification protocol given to + // `Notifications` + let (protocol_configs, handles): (Vec<_>, Vec<_>) = iter::once({ + let config = notifications::ProtocolConfig { + name: block_announces_protocol.protocol_name().clone(), + fallback_names: block_announces_protocol.fallback_names().cloned().collect(), + handshake: block_announces_protocol.handshake().as_ref().unwrap().to_vec(), + max_notification_size: block_announces_protocol.max_notification_size(), + }; + + let (handle, command_stream) = + block_announces_protocol.take_protocol_handle().split(); + + ((config, handle.clone(), command_stream), handle) + }) + .chain(notification_protocols.into_iter().map(|s| { + let config = notifications::ProtocolConfig { + name: s.protocol_name().clone(), + fallback_names: s.fallback_names().cloned().collect(), + handshake: s.handshake().as_ref().map_or(roles.encode(), |h| (*h).to_vec()), + max_notification_size: s.max_notification_size(), + }; + + let (handle, command_stream) = s.take_protocol_handle().split(); + + ((config, handle.clone(), command_stream), handle) + })) + .unzip(); + + ( + Notifications::new( + protocol_controller_handles, + from_protocol_controllers, + registry, + protocol_configs.into_iter(), + ), + installed_protocols, + handles, ) }; let protocol = Self { - peer_store_handle, behaviour, - notification_protocols: iter::once(block_announces_protocol.notifications_protocol) - .chain(notification_protocols.iter().map(|s| s.notifications_protocol.clone())) - .collect(), - bad_handshake_substreams: Default::default(), - peers: HashMap::new(), - sync_substream_validations: FuturesUnordered::new(), - tx, + sync_handle: handles[0].clone(), + peer_store_handle, + notification_protocols, + bad_handshake_streams: HashSet::new(), // TODO: remove when `BlockAnnouncesHandshake` is moved away from `Protocol` _marker: Default::default(), }; - Ok(protocol) + Ok((protocol, handles)) + } + + pub fn num_sync_peers(&self) -> usize { + self.sync_handle.num_peers() } /// Returns the list of all the peers we have an open channel to. @@ -163,21 +169,12 @@ impl Protocol { } } - /// Returns the number of peers we're connected to on sync protocol. - pub fn num_connected_peers(&self) -> usize { - self.peers.len() - } - - /// Set handshake for the notification protocol. - pub fn set_notification_handshake(&mut self, protocol: ProtocolName, handshake: Vec) { - if let Some(index) = self.notification_protocols.iter().position(|p| *p == protocol) { - self.behaviour.set_notif_protocol_handshake(SetId::from(index), handshake); - } else { - error!( - target: "sub-libp2p", - "set_notification_handshake with unknown protocol: {}", - protocol - ); + /// Check if role is available for `peer_id` by attempt to decode the handshake to roles and if + /// that fails, check if the role has been registered to `PeerStore`. + fn role_available(&self, peer_id: &PeerId, handshake: &Vec) -> bool { + match Roles::decode_all(&mut &handshake[..]) { + Ok(_) => true, + Err(_) => self.peer_store_handle.peer_role(&peer_id).is_some(), } } } @@ -189,25 +186,42 @@ pub enum CustomMessageOutcome { /// Notification protocols have been opened with a remote. NotificationStreamOpened { remote: PeerId, - protocol: ProtocolName, + // protocol: ProtocolName, + set_id: SetId, + /// Direction of the stream. + direction: Direction, /// See [`crate::Event::NotificationStreamOpened::negotiated_fallback`]. negotiated_fallback: Option, - roles: Roles, + /// Received handshake. received_handshake: Vec, + /// Notification sink. notifications_sink: NotificationsSink, }, /// The [`NotificationsSink`] of some notification protocols need an update. NotificationStreamReplaced { + // Peer ID. remote: PeerId, - protocol: ProtocolName, + /// Set ID. + set_id: SetId, + /// New notification sink. notifications_sink: NotificationsSink, }, /// Notification protocols have been closed with a remote. - NotificationStreamClosed { remote: PeerId, protocol: ProtocolName }, + NotificationStreamClosed { + // Peer ID. + remote: PeerId, + /// Set ID. + set_id: SetId, + }, /// Messages have been received on one or more notifications protocols. - NotificationsReceived { remote: PeerId, messages: Vec<(ProtocolName, Bytes)> }, - /// Now connected to a new peer for syncing purposes. - None, + NotificationsReceived { + // Peer ID. + remote: PeerId, + /// Set ID. + set_id: SetId, + /// Received notification. + notification: Vec, + }, } impl NetworkBehaviour for Protocol { @@ -274,23 +288,6 @@ impl NetworkBehaviour for Protocol { cx: &mut std::task::Context, params: &mut impl PollParameters, ) -> Poll>> { - while let Poll::Ready(Some(validation_result)) = - self.sync_substream_validations.poll_next_unpin(cx) - { - match validation_result { - Ok((peer, roles)) => { - self.peers.insert(peer, roles); - }, - Err(peer) => { - log::debug!( - target: "sub-libp2p", - "`SyncingEngine` rejected stream" - ); - self.behaviour.disconnect_peer(&peer, HARDCODED_PEERSETS_SYNC); - }, - } - } - let event = match self.behaviour.poll(cx, params) { Poll::Pending => return Poll::Pending, Poll::Ready(ToSwarm::GenerateEvent(ev)) => ev, @@ -307,204 +304,86 @@ impl NetworkBehaviour for Protocol { NotificationsOut::CustomProtocolOpen { peer_id, set_id, + direction, received_handshake, notifications_sink, negotiated_fallback, - inbound, - } => { - // Set number 0 is hardcoded the default set of peers we sync from. + .. + } => if set_id == HARDCODED_PEERSETS_SYNC { - // `received_handshake` can be either a `Status` message if received from the - // legacy substream ,or a `BlockAnnouncesHandshake` if received from the block - // announces substream. - match as DecodeAll>::decode_all(&mut &received_handshake[..]) { - Ok(GenericMessage::Status(handshake)) => { - let roles = handshake.roles; - let handshake = BlockAnnouncesHandshake:: { - roles: handshake.roles, - best_number: handshake.best_number, - best_hash: handshake.best_hash, - genesis_hash: handshake.genesis_hash, - }; - - let (tx, rx) = oneshot::channel(); - let _ = self.tx.unbounded_send( - crate::SyncEvent::NotificationStreamOpened { - inbound, - remote: peer_id, - received_handshake: handshake, - sink: notifications_sink, - tx, - }, - ); - self.sync_substream_validations.push(Box::pin(async move { - match rx.await { - Ok(accepted) => - if accepted { - Ok((peer_id, roles)) - } else { - Err(peer_id) - }, - Err(_) => Err(peer_id), - } - })); - - CustomMessageOutcome::None - }, - Ok(msg) => { - debug!( - target: "sync", - "Expected Status message from {}, but got {:?}", - peer_id, - msg, - ); - self.peer_store_handle.report_peer(peer_id, rep::BAD_MESSAGE); - CustomMessageOutcome::None - }, - Err(err) => { - match as DecodeAll>::decode_all( - &mut &received_handshake[..], - ) { - Ok(handshake) => { - let roles = handshake.roles; - - let (tx, rx) = oneshot::channel(); - let _ = self.tx.unbounded_send( - crate::SyncEvent::NotificationStreamOpened { - inbound, - remote: peer_id, - received_handshake: handshake, - sink: notifications_sink, - tx, - }, - ); - self.sync_substream_validations.push(Box::pin(async move { - match rx.await { - Ok(accepted) => - if accepted { - Ok((peer_id, roles)) - } else { - Err(peer_id) - }, - Err(_) => Err(peer_id), - } - })); - CustomMessageOutcome::None - }, - Err(err2) => { - log::debug!( - target: "sync", - "Couldn't decode handshake sent by {}: {:?}: {} & {}", - peer_id, - received_handshake, - err, - err2, - ); - self.peer_store_handle.report_peer(peer_id, rep::BAD_MESSAGE); - CustomMessageOutcome::None - }, - } - }, - } + let _ = self.sync_handle.report_substream_opened( + peer_id, + direction, + received_handshake, + negotiated_fallback, + notifications_sink, + ); + None } else { - match ( - Roles::decode_all(&mut &received_handshake[..]), - self.peers.get(&peer_id), - ) { - (Ok(roles), _) => CustomMessageOutcome::NotificationStreamOpened { + match self.role_available(&peer_id, &received_handshake) { + true => Some(CustomMessageOutcome::NotificationStreamOpened { remote: peer_id, - protocol: self.notification_protocols[usize::from(set_id)].clone(), + set_id, + direction, negotiated_fallback, - roles, received_handshake, notifications_sink, - }, - (Err(_), Some(roles)) if received_handshake.is_empty() => { - // As a convenience, we allow opening substreams for "external" - // notification protocols with an empty handshake. This fetches the - // roles from the locally-known roles. - // TODO: remove this after https://github.com/paritytech/substrate/issues/5685 - CustomMessageOutcome::NotificationStreamOpened { - remote: peer_id, - protocol: self.notification_protocols[usize::from(set_id)].clone(), - negotiated_fallback, - roles: *roles, - received_handshake, - notifications_sink, - } - }, - (Err(err), _) => { - debug!(target: "sync", "Failed to parse remote handshake: {}", err); - self.bad_handshake_substreams.insert((peer_id, set_id)); - self.behaviour.disconnect_peer(&peer_id, set_id); - self.peer_store_handle.report_peer(peer_id, rep::BAD_MESSAGE); - CustomMessageOutcome::None + }), + false => { + self.bad_handshake_streams.insert(peer_id); + None }, } - } - }, + }, NotificationsOut::CustomProtocolReplaced { peer_id, notifications_sink, set_id } => - if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { - CustomMessageOutcome::None - } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationSinkReplaced { - remote: peer_id, - sink: notifications_sink, - }); - CustomMessageOutcome::None + if set_id == HARDCODED_PEERSETS_SYNC { + let _ = self + .sync_handle + .report_notification_sink_replaced(peer_id, notifications_sink); + None } else { - CustomMessageOutcome::NotificationStreamReplaced { - remote: peer_id, - protocol: self.notification_protocols[usize::from(set_id)].clone(), - notifications_sink, - } + (!self.bad_handshake_streams.contains(&peer_id)).then_some( + CustomMessageOutcome::NotificationStreamReplaced { + remote: peer_id, + set_id, + notifications_sink, + }, + ) }, NotificationsOut::CustomProtocolClosed { peer_id, set_id } => { - if self.bad_handshake_substreams.remove(&(peer_id, set_id)) { - // The substream that has just been closed had been opened with a bad - // handshake. The outer layers have never received an opening event about this - // substream, and consequently shouldn't receive a closing event either. - CustomMessageOutcome::None - } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationStreamClosed { - remote: peer_id, - }); - self.peers.remove(&peer_id); - CustomMessageOutcome::None + if set_id == HARDCODED_PEERSETS_SYNC { + let _ = self.sync_handle.report_substream_closed(peer_id); + None } else { - CustomMessageOutcome::NotificationStreamClosed { - remote: peer_id, - protocol: self.notification_protocols[usize::from(set_id)].clone(), - } + (!self.bad_handshake_streams.remove(&peer_id)).then_some( + CustomMessageOutcome::NotificationStreamClosed { remote: peer_id, set_id }, + ) } }, NotificationsOut::Notification { peer_id, set_id, message } => { - if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { - CustomMessageOutcome::None - } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationsReceived { - remote: peer_id, - messages: vec![message.freeze()], - }); - CustomMessageOutcome::None + if set_id == HARDCODED_PEERSETS_SYNC { + let _ = self + .sync_handle + .report_notification_received(peer_id, message.freeze().into()); + None } else { - let protocol_name = self.notification_protocols[usize::from(set_id)].clone(); - CustomMessageOutcome::NotificationsReceived { - remote: peer_id, - messages: vec![(protocol_name, message.freeze())], - } + (!self.bad_handshake_streams.contains(&peer_id)).then_some( + CustomMessageOutcome::NotificationsReceived { + remote: peer_id, + set_id, + notification: message.freeze().into(), + }, + ) } }, }; - if !matches!(outcome, CustomMessageOutcome::None) { - return Poll::Ready(ToSwarm::GenerateEvent(outcome)) + match outcome { + Some(event) => Poll::Ready(ToSwarm::GenerateEvent(event)), + None => { + cx.waker().wake_by_ref(); + Poll::Pending + }, } - - // This block can only be reached if an event was pulled from the behaviour and that - // resulted in `CustomMessageOutcome::None`. Since there might be another pending - // message from the behaviour, the task is scheduled again. - cx.waker().wake_by_ref(); - Poll::Pending } } diff --git a/substrate/client/network/src/protocol/message.rs b/substrate/client/network/src/protocol/message.rs index 66dca2975375..247580083f99 100644 --- a/substrate/client/network/src/protocol/message.rs +++ b/substrate/client/network/src/protocol/message.rs @@ -29,6 +29,7 @@ use sc_network_common::message::RequestId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; /// Type alias for using the message type using block type parameters. +#[allow(unused)] pub type Message = generic::Message< ::Header, ::Hash, diff --git a/substrate/client/network/src/protocol/notifications.rs b/substrate/client/network/src/protocol/notifications.rs index aa49cfcf9d44..10fa329097d1 100644 --- a/substrate/client/network/src/protocol/notifications.rs +++ b/substrate/client/network/src/protocol/notifications.rs @@ -22,9 +22,13 @@ pub use self::{ behaviour::{Notifications, NotificationsOut, ProtocolConfig}, handler::{NotificationsSink, NotifsHandlerError, Ready}, + service::{notification_service, ProtocolHandlePair}, }; +pub(crate) use self::service::ProtocolHandle; + mod behaviour; mod handler; +mod service; mod tests; mod upgrade; diff --git a/substrate/client/network/src/protocol/notifications/behaviour.rs b/substrate/client/network/src/protocol/notifications/behaviour.rs index b78f15f8529c..ef0c6540eee5 100644 --- a/substrate/client/network/src/protocol/notifications/behaviour.rs +++ b/substrate/client/network/src/protocol/notifications/behaviour.rs @@ -17,16 +17,18 @@ // along with this program. If not, see . use crate::{ - protocol::notifications::handler::{ - self, NotificationsSink, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut, + protocol::notifications::{ + handler::{self, NotificationsSink, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut}, + service::{metrics, NotificationCommand, ProtocolHandle, ValidationCallResult}, }, protocol_controller::{self, IncomingIndex, Message, SetId}, + service::traits::{Direction, ValidationResult}, types::ProtocolName, }; use bytes::BytesMut; use fnv::FnvHashMap; -use futures::prelude::*; +use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; use libp2p::{ core::{ConnectedPoint, Endpoint, Multiaddr}, swarm::{ @@ -36,11 +38,15 @@ use libp2p::{ }, PeerId, }; -use log::{debug, error, info, trace, warn}; +use log::{debug, error, trace, warn}; use parking_lot::RwLock; +use prometheus_endpoint::Registry; use rand::distributions::{Distribution as _, Uniform}; use sc_utils::mpsc::TracingUnboundedReceiver; use smallvec::SmallVec; +use tokio::sync::oneshot::error::RecvError; +use tokio_stream::StreamMap; + use std::{ cmp, collections::{hash_map::Entry, VecDeque}, @@ -51,6 +57,13 @@ use std::{ time::{Duration, Instant}, }; +/// Type representing a pending substream validation. +type PendingInboundValidation = + BoxFuture<'static, (Result, IncomingIndex)>; + +/// Logging target for the file. +const LOG_TARGET: &str = "sub-libp2p"; + /// Network behaviour that handles opening substreams for custom protocols with other peers. /// /// # How it works @@ -106,6 +119,12 @@ pub struct Notifications { /// Notification protocols. Entries never change after initialization. notif_protocols: Vec, + /// Protocol handles. + protocol_handles: Vec, + + // Command streams. + command_streams: StreamMap + Send + Unpin>>, + /// Protocol controllers are responsible for peer connections management. protocol_controller_handles: Vec, @@ -138,6 +157,18 @@ pub struct Notifications { /// Events to produce from `poll()`. events: VecDeque>, + + /// Pending inbound substream validations. + // + // NOTE: it's possible to read a stale response from `pending_inbound_validations` + // as the substream may get closed by the remote peer before the protocol has had + // a chance to validate it. [`Notifications`] must compare the `crate::peerset::IncomingIndex` + // returned by the completed future against the `crate::peerset::IncomingIndex` stored in + // `PeerState::Incoming` to check whether the completed future is stale or not. + pending_inbound_validations: FuturesUnordered, + + /// Metrics for notifications. + metrics: Option, } /// Configuration for a notifications protocol. @@ -235,6 +266,9 @@ enum PeerState { /// Incoming index tracking this connection. incoming_index: IncomingIndex, + /// Peerset has signaled it wants the substream closed. + peerset_rejected: bool, + /// List of connections with this peer, and their state. connections: SmallVec<[(ConnectionId, ConnectionState); crate::MAX_CONNECTIONS_PER_PEER]>, }, @@ -303,6 +337,8 @@ struct IncomingPeer { alive: bool, /// Id that the we sent to the peerset. incoming_id: IncomingIndex, + /// Received handshake. + handshake: Vec, } /// Event that can be emitted by the `Notifications`. @@ -314,6 +350,8 @@ pub enum NotificationsOut { peer_id: PeerId, /// Peerset set ID the substream is tied to. set_id: SetId, + /// Direction of the stream. + direction: Direction, /// If `Some`, a fallback protocol name has been used rather the main protocol name. /// Always matches one of the fallback names passed at initialization. negotiated_fallback: Option, @@ -364,24 +402,52 @@ pub enum NotificationsOut { impl Notifications { /// Creates a `CustomProtos`. - pub fn new( + pub(crate) fn new( protocol_controller_handles: Vec, from_protocol_controllers: TracingUnboundedReceiver, - notif_protocols: impl Iterator, + registry: &Option, + notif_protocols: impl Iterator< + Item = ( + ProtocolConfig, + ProtocolHandle, + Box + Send + Unpin>, + ), + >, ) -> Self { - let notif_protocols = notif_protocols - .map(|cfg| handler::ProtocolConfig { - name: cfg.name, - fallback_names: cfg.fallback_names, - handshake: Arc::new(RwLock::new(cfg.handshake)), - max_notification_size: cfg.max_notification_size, + let (notif_protocols, protocol_handles): (Vec<_>, Vec<_>) = notif_protocols + .map(|(cfg, protocol_handle, command_stream)| { + ( + handler::ProtocolConfig { + name: cfg.name, + fallback_names: cfg.fallback_names, + handshake: Arc::new(RwLock::new(cfg.handshake)), + max_notification_size: cfg.max_notification_size, + }, + (protocol_handle, command_stream), + ) }) - .collect::>(); - + .unzip(); assert!(!notif_protocols.is_empty()); + let metrics = registry.as_ref().and_then(|registry| metrics::register(®istry).ok()); + let (mut protocol_handles, command_streams): (Vec<_>, Vec<_>) = protocol_handles + .into_iter() + .enumerate() + .map(|(set_id, (mut protocol_handle, command_stream))| { + protocol_handle.set_metrics(metrics.clone()); + + (protocol_handle, (set_id, command_stream)) + }) + .unzip(); + + protocol_handles.iter_mut().skip(1).for_each(|handle| { + handle.delegate_to_peerset(true); + }); + Self { notif_protocols, + protocol_handles, + command_streams: StreamMap::from_iter(command_streams.into_iter()), protocol_controller_handles, from_protocol_controllers, peers: FnvHashMap::default(), @@ -390,6 +456,8 @@ impl Notifications { incoming: SmallVec::new(), next_incoming_index: IncomingIndex(0), events: VecDeque::new(), + pending_inbound_validations: FuturesUnordered::new(), + metrics, } } @@ -807,14 +875,21 @@ impl Notifications { *entry.into_mut() = PeerState::Backoff { timer, timer_deadline } }, - // Invalid state transitions. - st @ PeerState::Incoming { .. } => { - info!( + // `ProtocolController` disconnected peer while it was still being validated by the + // protocol, mark the connection as rejected and once the validation is received from + // the protocol, reject the substream + PeerState::Incoming { backoff_until, connections, incoming_index, .. } => { + debug!( target: "sub-libp2p", "PSM => Drop({}, {:?}): Ignoring obsolete disconnect, we are awaiting accept/reject.", entry.key().0, set_id, ); - *entry.into_mut() = st; + *entry.into_mut() = PeerState::Incoming { + backoff_until, + connections, + incoming_index, + peerset_rejected: true, + }; }, PeerState::Poisoned => { error!(target: "sub-libp2p", "State of {:?} is poisoned", entry.key()); @@ -823,20 +898,71 @@ impl Notifications { } } + /// Substream has been accepted by the `ProtocolController` and must now be sent + /// to the protocol for validation. + fn peerset_report_preaccept(&mut self, index: IncomingIndex) { + let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) else { + error!(target: LOG_TARGET, "PSM => Preaccept({:?}): Invalid index", index); + return + }; + + trace!( + target: LOG_TARGET, + "PSM => Preaccept({:?}): Sent to protocol for validation", + index + ); + let incoming = &self.incoming[pos]; + + match self.protocol_handles[usize::from(incoming.set_id)] + .report_incoming_substream(incoming.peer_id, incoming.handshake.clone()) + { + Ok(ValidationCallResult::Delegated) => { + self.protocol_report_accept(index); + }, + Ok(ValidationCallResult::WaitForValidation(rx)) => { + self.pending_inbound_validations + .push(Box::pin(async move { (rx.await, index) })); + }, + Err(err) => { + // parachain collators enable the syncing protocol but `NotificationService` for + // `SyncingEngine` is not created which causes `report_incoming_substream()` to + // fail. This is not a fatal error and should be ignored even though in typical + // cases the `NotificationService` not existing is a fatal error and indicates that + // the protocol has exited. Until the parachain collator issue is fixed, just report + // and error and reject the peer. + debug!(target: LOG_TARGET, "protocol has exited: {err:?} {:?}", incoming.set_id); + + self.protocol_report_reject(index); + }, + } + } + /// Function that is called when the peerset wants us to accept a connection /// request from a peer. - fn peerset_report_accept(&mut self, index: IncomingIndex) { - let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) - { - self.incoming.remove(pos) - } else { - error!(target: "sub-libp2p", "PSM => Accept({:?}): Invalid index", index); - return + fn protocol_report_accept(&mut self, index: IncomingIndex) { + let (pos, incoming) = + if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { + (pos, self.incoming.get(pos)) + } else { + error!(target: "sub-libp2p", "PSM => Accept({:?}): Invalid index", index); + return + }; + + let Some(incoming) = incoming else { + error!(target: "sub-libp2p", "Incoming connection ({:?}) doesn't exist", index); + debug_assert!(false); + return; }; if !incoming.alive { - trace!(target: "sub-libp2p", "PSM => Accept({:?}, {}, {:?}): Obsolete incoming", - index, incoming.peer_id, incoming.set_id); + trace!( + target: "sub-libp2p", + "PSM => Accept({:?}, {}, {:?}): Obsolete incoming", + index, + incoming.peer_id, + incoming.set_id, + ); + match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(PeerState::DisabledPendingEnable { .. }) | Some(PeerState::Enabled { .. }) => { }, @@ -847,26 +973,42 @@ impl Notifications { .dropped(incoming.peer_id); }, } + + self.incoming.remove(pos); return } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { - debug_assert!(false); + log::debug!( + target: "sub-libp2p", + "Connection to {:?} closed, ({:?} {:?}), ignoring accept", + incoming.peer_id, + incoming.set_id, + index, + ); + self.incoming.remove(pos); return }, }; match mem::replace(state, PeerState::Poisoned) { // Incoming => Enabled - PeerState::Incoming { mut connections, incoming_index, .. } => { + PeerState::Incoming { + mut connections, + incoming_index, + peerset_rejected, + backoff_until, + } => { if index < incoming_index { warn!( target: "sub-libp2p", "PSM => Accept({:?}, {}, {:?}): Ignoring obsolete incoming index, we are already awaiting {:?}.", index, incoming.peer_id, incoming.set_id, incoming_index ); + + self.incoming.remove(pos); return } else if index > incoming_index { error!( @@ -874,12 +1016,39 @@ impl Notifications { "PSM => Accept({:?}, {}, {:?}): Ignoring incoming index from the future, we are awaiting {:?}.", index, incoming.peer_id, incoming.set_id, incoming_index ); + + self.incoming.remove(pos); debug_assert!(false); return } - trace!(target: "sub-libp2p", "PSM => Accept({:?}, {}, {:?}): Enabling connections.", - index, incoming.peer_id, incoming.set_id); + // while the substream was being validated by the protocol, `Peerset` had request + // for the it to be closed so reject the substream now + if peerset_rejected { + trace!( + target: "sub-libp2p", + "Protocol accepted ({:?} {:?} {:?}) but Peerset had request disconnection, rejecting", + index, + incoming.peer_id, + incoming.set_id + ); + + *state = PeerState::Incoming { + connections, + backoff_until, + peerset_rejected, + incoming_index, + }; + return self.report_reject(index).map_or((), |_| ()); + } + + trace!( + target: "sub-libp2p", + "PSM => Accept({:?}, {}, {:?}): Enabling connections.", + index, + incoming.peer_id, + incoming.set_id + ); debug_assert!(connections .iter() @@ -898,53 +1067,85 @@ impl Notifications { *connec_state = ConnectionState::Opening; } + self.incoming.remove(pos); *state = PeerState::Enabled { connections }; }, - + st @ PeerState::Disabled { .. } | st @ PeerState::Backoff { .. } => { + self.incoming.remove(pos); + *state = st; + }, // Any state other than `Incoming` is invalid. peer => { - error!(target: "sub-libp2p", + error!( + target: "sub-libp2p", "State mismatch in libp2p: Expected alive incoming. Got {:?}.", - peer); + peer + ); + + self.incoming.remove(pos); debug_assert!(false); }, } } - /// Function that is called when the peerset wants us to reject an incoming peer. + /// Function that is called when `ProtocolController` wants us to reject an incoming peer. fn peerset_report_reject(&mut self, index: IncomingIndex) { + let _ = self.report_reject(index); + } + + /// Function that is called when the protocol wants us to reject an incoming peer. + fn protocol_report_reject(&mut self, index: IncomingIndex) { + if let Some((set_id, peer_id)) = self.report_reject(index) { + self.protocol_controller_handles[usize::from(set_id)].dropped(peer_id) + } + } + + /// Function that is called when the peerset wants us to reject an incoming peer. + fn report_reject(&mut self, index: IncomingIndex) -> Option<(SetId, PeerId)> { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) } else { error!(target: "sub-libp2p", "PSM => Reject({:?}): Invalid index", index); - return + return None }; if !incoming.alive { - trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Obsolete incoming, \ - ignoring", index, incoming.peer_id, incoming.set_id); - return + trace!( + target: "sub-libp2p", + "PSM => Reject({:?}, {}, {:?}): Obsolete incoming, ignoring", + index, + incoming.peer_id, + incoming.set_id, + ); + + return None } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { - debug_assert!(false); - return + log::debug!( + target: "sub-libp2p", + "Connection to {:?} closed, ({:?} {:?}), ignoring accept", + incoming.peer_id, + incoming.set_id, + index, + ); + return None }, }; match mem::replace(state, PeerState::Poisoned) { // Incoming => Disabled - PeerState::Incoming { mut connections, backoff_until, incoming_index } => { + PeerState::Incoming { mut connections, backoff_until, incoming_index, .. } => { if index < incoming_index { warn!( target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Ignoring obsolete incoming index, we are already awaiting {:?}.", index, incoming.peer_id, incoming.set_id, incoming_index ); - return + return None } else if index > incoming_index { error!( target: "sub-libp2p", @@ -952,7 +1153,7 @@ impl Notifications { index, incoming.peer_id, incoming.set_id, incoming_index ); debug_assert!(false); - return + return None } trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Rejecting connections.", @@ -976,10 +1177,20 @@ impl Notifications { } *state = PeerState::Disabled { connections, backoff_until }; + Some((incoming.set_id, incoming.peer_id)) + }, + // connection to peer may have been closed already + st @ PeerState::Disabled { .. } | st @ PeerState::Backoff { .. } => { + *state = st; + None + }, + peer => { + error!( + target: LOG_TARGET, + "State mismatch in libp2p: Expected alive incoming. Got {peer:?}.", + ); + None }, - peer => error!(target: "sub-libp2p", - "State mismatch in libp2p: Expected alive incoming. Got {:?}.", - peer), } } } @@ -1021,6 +1232,7 @@ impl NetworkBehaviour for Notifications { send_back_addr: remote_addr.clone(), }, self.notif_protocols.clone(), + self.metrics.clone(), )) } @@ -1035,6 +1247,7 @@ impl NetworkBehaviour for Notifications { peer, ConnectedPoint::Dialer { address: addr.clone(), role_override }, self.notif_protocols.clone(), + self.metrics.clone(), )) } @@ -1195,7 +1408,12 @@ impl NetworkBehaviour for Notifications { }, // Incoming => Incoming | Disabled | Backoff | Ø - PeerState::Incoming { mut connections, backoff_until, incoming_index } => { + PeerState::Incoming { + mut connections, + backoff_until, + incoming_index, + peerset_rejected, + } => { trace!( target: "sub-libp2p", "Libp2p => Disconnected({}, {:?}, {:?}): OpenDesiredByRemote.", @@ -1274,6 +1492,7 @@ impl NetworkBehaviour for Notifications { connections, backoff_until, incoming_index, + peerset_rejected, }; } }, @@ -1313,7 +1532,7 @@ impl NetworkBehaviour for Notifications { let event = NotificationsOut::CustomProtocolReplaced { peer_id, set_id, - notifications_sink: replacement_sink, + notifications_sink: replacement_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); } @@ -1474,7 +1693,7 @@ impl NetworkBehaviour for Notifications { event: THandlerOutEvent, ) { match event { - NotifsHandlerOut::OpenDesiredByRemote { protocol_index } => { + NotifsHandlerOut::OpenDesiredByRemote { protocol_index, handshake } => { let set_id = SetId::from(protocol_index); trace!(target: "sub-libp2p", @@ -1495,7 +1714,12 @@ impl NetworkBehaviour for Notifications { match mem::replace(entry.get_mut(), PeerState::Poisoned) { // Incoming => Incoming - PeerState::Incoming { mut connections, backoff_until, incoming_index } => { + PeerState::Incoming { + mut connections, + backoff_until, + incoming_index, + peerset_rejected, + } => { debug_assert!(connections .iter() .any(|(_, s)| matches!(s, ConnectionState::OpenDesiredByRemote))); @@ -1523,8 +1747,12 @@ impl NetworkBehaviour for Notifications { debug_assert!(false); } - *entry.into_mut() = - PeerState::Incoming { connections, backoff_until, incoming_index }; + *entry.into_mut() = PeerState::Incoming { + connections, + backoff_until, + incoming_index, + peerset_rejected, + }; }, PeerState::Enabled { mut connections } => { @@ -1588,11 +1816,13 @@ impl NetworkBehaviour for Notifications { set_id, alive: true, incoming_id, + handshake, }); *entry.into_mut() = PeerState::Incoming { connections, backoff_until, + peerset_rejected: false, incoming_index: incoming_id, }; } else { @@ -1725,7 +1955,7 @@ impl NetworkBehaviour for Notifications { let event = NotificationsOut::CustomProtocolReplaced { peer_id, set_id, - notifications_sink: replacement_sink, + notifications_sink: replacement_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); } @@ -1830,8 +2060,13 @@ impl NetworkBehaviour for Notifications { peer_id, set_id, inbound, - negotiated_fallback, - received_handshake, + direction: if inbound { + Direction::Inbound + } else { + Direction::Outbound + }, + received_handshake: received_handshake.clone(), + negotiated_fallback: negotiated_fallback.clone(), notifications_sink: notifications_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); @@ -1979,8 +2214,11 @@ impl NetworkBehaviour for Notifications { peer_id, set_id, ); - let event = NotificationsOut::Notification { peer_id, set_id, message }; - + let event = NotificationsOut::Notification { + peer_id, + set_id, + message: message.clone(), + }; self.events.push_back(ToSwarm::GenerateEvent(event)); } else { trace!( @@ -2009,10 +2247,10 @@ impl NetworkBehaviour for Notifications { loop { match futures::Stream::poll_next(Pin::new(&mut self.from_protocol_controllers), cx) { Poll::Ready(Some(Message::Accept(index))) => { - self.peerset_report_accept(index); + self.peerset_report_preaccept(index); }, Poll::Ready(Some(Message::Reject(index))) => { - self.peerset_report_reject(index); + let _ = self.peerset_report_reject(index); }, Poll::Ready(Some(Message::Connect { peer_id, set_id, .. })) => { self.peerset_report_connect(peer_id, set_id); @@ -2031,6 +2269,43 @@ impl NetworkBehaviour for Notifications { } } + // poll commands from protocols + loop { + match futures::Stream::poll_next(Pin::new(&mut self.command_streams), cx) { + Poll::Ready(Some((set_id, command))) => match command { + NotificationCommand::SetHandshake(handshake) => { + self.set_notif_protocol_handshake(set_id.into(), handshake); + }, + NotificationCommand::OpenSubstream(_peer) | + NotificationCommand::CloseSubstream(_peer) => { + todo!("substream control not implemented"); + }, + }, + Poll::Ready(None) => { + error!(target: LOG_TARGET, "Protocol command streams have been shut down"); + break + }, + Poll::Pending => break, + } + } + + while let Poll::Ready(Some((result, index))) = + self.pending_inbound_validations.poll_next_unpin(cx) + { + match result { + Ok(ValidationResult::Accept) => { + self.protocol_report_accept(index); + }, + Ok(ValidationResult::Reject) => { + self.protocol_report_reject(index); + }, + Err(_) => { + error!(target: LOG_TARGET, "Protocol has shut down"); + break + }, + } + } + while let Poll::Ready(Some((delay_id, peer_id, set_id))) = Pin::new(&mut self.delays).poll_next(cx) { @@ -2153,7 +2428,10 @@ mod tests { } } - fn development_notifs() -> (Notifications, ProtocolController) { + fn development_notifs( + ) -> (Notifications, ProtocolController, Box) { + let (protocol_handle_pair, notif_service) = + crate::protocol::notifications::service::notification_service("/proto/1".into()); let (to_notifications, from_controller) = tracing_unbounded("test_controller_to_notifications", 10_000); @@ -2169,24 +2447,31 @@ mod tests { Box::new(MockPeerStore {}), ); + let (notif_handle, command_stream) = protocol_handle_pair.split(); ( Notifications::new( vec![handle], from_controller, - iter::once(ProtocolConfig { - name: "/foo".into(), - fallback_names: Vec::new(), - handshake: vec![1, 2, 3, 4], - max_notification_size: u64::MAX, - }), + &None, + iter::once(( + ProtocolConfig { + name: "/foo".into(), + fallback_names: Vec::new(), + handshake: vec![1, 2, 3, 4], + max_notification_size: u64::MAX, + }, + notif_handle, + command_stream, + )), ), controller, + notif_service, ) } #[test] fn update_handshake() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let inner = notif.notif_protocols.get_mut(0).unwrap().handshake.read().clone(); assert_eq!(inner, vec![1, 2, 3, 4]); @@ -2201,14 +2486,14 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn update_unknown_handshake() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); notif.set_notif_protocol_handshake(1337.into(), vec![5, 6, 7, 8]); } #[test] fn disconnect_backoff_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( @@ -2225,7 +2510,7 @@ mod tests { #[test] fn disconnect_pending_request() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( @@ -2242,7 +2527,7 @@ mod tests { #[test] fn disconnect_requested_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert((peer, 0.into()), PeerState::Requested); @@ -2253,7 +2538,7 @@ mod tests { #[test] fn disconnect_disabled_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( (peer, 0.into()), @@ -2269,7 +2554,7 @@ mod tests { #[test] fn remote_opens_connection_and_substream() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { @@ -2299,7 +2584,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(&PeerState::Incoming { ref connections, backoff_until: None, .. }) = @@ -2319,7 +2607,7 @@ mod tests { #[tokio::test] async fn disconnect_remote_substream_before_handled_by_controller() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { @@ -2339,7 +2627,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); notif.disconnect_peer(&peer, 0.into()); @@ -2355,7 +2646,7 @@ mod tests { #[test] fn peerset_report_connect_backoff() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2393,7 +2684,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2420,7 +2711,7 @@ mod tests { #[test] fn peerset_connect_incoming() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2444,19 +2735,22 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); // attempt to connect to the peer and verify that the peer state is `Enabled`; // we rely on implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); } #[test] fn peerset_disconnect_disable_pending_enable() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2503,7 +2797,7 @@ mod tests { #[test] fn peerset_disconnect_enabled() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2525,11 +2819,14 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // disconnect peer and verify that the state is `Disabled` @@ -2539,7 +2836,7 @@ mod tests { #[test] fn peerset_disconnect_requested() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); @@ -2554,7 +2851,7 @@ mod tests { #[test] fn peerset_disconnect_pending_request() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2587,7 +2884,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2607,7 +2904,7 @@ mod tests { #[test] fn peerset_accept_peer_not_alive() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2631,7 +2928,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2647,14 +2947,14 @@ mod tests { IncomingPeer { alive: false, incoming_id: IncomingIndex(0), .. }, )); - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert_eq!(notif.incoming.len(), 0); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(PeerState::Disabled { .. }))); } #[test] fn secondary_connection_peer_state_incoming() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -2678,7 +2978,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Incoming { connections, .. }) = notif.peers.get(&(peer, set_id)) { assert_eq!(connections.len(), 1); @@ -2709,7 +3012,7 @@ mod tests { #[test] fn close_connection_for_disabled_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2734,7 +3037,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2743,7 +3046,7 @@ mod tests { #[test] fn close_connection_for_incoming_peer_one_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2766,7 +3069,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2775,7 +3081,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2788,7 +3094,7 @@ mod tests { #[test] fn close_connection_for_incoming_peer_two_connections() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let conn1 = ConnectionId::new_unchecked(1); @@ -2815,7 +3121,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2842,7 +3151,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2857,7 +3166,7 @@ mod tests { #[test] fn connection_and_substream_open() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2882,13 +3191,16 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); // We rely on the implementation detail that incoming indices are counted // from 0 to not mock the `Peerset`. - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // open new substream @@ -2911,7 +3223,7 @@ mod tests { #[test] fn connection_closed_sink_replaced() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -2947,7 +3259,10 @@ mod tests { notif.on_connection_handler_event( peer, conn2, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Enabled { connections, .. }) = notif.peers.get(&(peer, set_id)) { @@ -2984,7 +3299,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3005,7 +3320,7 @@ mod tests { #[test] fn dial_failure_for_requested_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); @@ -3028,7 +3343,7 @@ mod tests { #[tokio::test] async fn write_notification() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -3077,7 +3392,7 @@ mod tests { #[test] fn peerset_report_connect_backoff_expired() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3110,7 +3425,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3125,7 +3440,7 @@ mod tests { #[test] fn peerset_report_disconnect_disabled() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3151,7 +3466,7 @@ mod tests { #[test] fn peerset_report_disconnect_backoff() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3184,7 +3499,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3197,7 +3512,7 @@ mod tests { #[test] fn peer_is_backed_off_if_both_connections_get_closed_while_peer_is_disabled_with_back_off() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); @@ -3247,7 +3562,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected.clone(), vec![]), + handler: NotifsHandler::new(peer, connected.clone(), vec![], None), remaining_established: 0usize, }, )); @@ -3261,7 +3576,7 @@ mod tests { peer_id: peer, connection_id: conn2, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3270,7 +3585,7 @@ mod tests { #[test] fn inject_connection_closed_incoming_with_backoff() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3294,7 +3609,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); // manually add backoff for the entry @@ -3312,7 +3630,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3321,7 +3639,7 @@ mod tests { #[test] fn two_connections_inactive_connection_gets_closed_peer_state_is_still_incoming() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3355,7 +3673,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!( notif.peers.get_mut(&(peer, 0.into())), @@ -3367,7 +3688,7 @@ mod tests { peer_id: peer, connection_id: conn2, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3376,7 +3697,7 @@ mod tests { #[test] fn two_connections_active_connection_gets_closed_peer_state_is_disabled() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3413,7 +3734,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!( notif.peers.get_mut(&(peer, 0.into())), @@ -3425,7 +3749,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3434,7 +3758,7 @@ mod tests { #[test] fn inject_connection_closed_for_active_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3494,7 +3818,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3502,7 +3826,7 @@ mod tests { #[test] fn inject_dial_failure_for_pending_request() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3535,7 +3859,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3565,7 +3889,7 @@ mod tests { #[test] fn peerstate_incoming_open_desired_by_remote() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn1 = ConnectionId::new_unchecked(0); @@ -3599,7 +3923,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -3607,7 +3934,10 @@ mod tests { notif.on_connection_handler_event( peer, conn2, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Incoming { ref connections, .. }) = notif.peers.get(&(peer, set_id)) @@ -3619,7 +3949,7 @@ mod tests { #[tokio::test] async fn remove_backoff_peer_after_timeout() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3652,7 +3982,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3697,7 +4027,7 @@ mod tests { #[tokio::test] async fn reschedule_disabled_pending_enable_when_connection_not_closed() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -3726,13 +4056,16 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); let event = conn_yielder.open_substream(peer, 0, connected, vec![1, 2, 3, 4]); @@ -3815,7 +4148,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_enabled_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -3840,7 +4173,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -3865,7 +4201,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_connect_with_disabled_pending_enable_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3911,7 +4247,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_connect_with_requested_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); @@ -3927,7 +4263,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_connect_with_pending_requested() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3960,7 +4296,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3984,7 +4320,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_connect_with_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -4008,7 +4344,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4019,7 +4358,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_disconnect_with_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -4043,7 +4382,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4052,13 +4394,68 @@ mod tests { } #[test] - #[should_panic] #[cfg(debug_assertions)] - fn peerset_report_accept_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + fn peerset_report_disconnect_with_incoming_peer_protocol_accepts() { + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); + let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); + let connected = ConnectedPoint::Listener { + local_addr: Multiaddr::empty(), + send_back_addr: Multiaddr::empty(), + }; + + notif.on_swarm_event(FromSwarm::ConnectionEstablished( + libp2p::swarm::behaviour::ConnectionEstablished { + peer_id: peer, + connection_id: conn, + endpoint: &connected, + failed_addresses: &[], + other_established: 0usize, + }, + )); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Disabled { .. }))); + + // remote opens a substream, verify that peer state is updated to `Incoming` + notif.on_connection_handler_event( + peer, + conn, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, + ); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); + + // `Peerset` wants to disconnect the peer but since it's still under validation, + // it won't be disabled automatically + notif.peerset_report_disconnect(peer, set_id); + + let incoming_index = match notif.peers.get(&(peer, set_id)) { + Some(&PeerState::Incoming { peerset_rejected, incoming_index, .. }) => { + assert!(peerset_rejected); + incoming_index + }, + state => panic!("invalid state: {state:?}"), + }; + + // protocol accepted peer but since `Peerset` wanted to disconnect it, the peer will be + // disabled + notif.protocol_report_accept(incoming_index); + + match notif.peers.get(&(peer, set_id)) { + Some(&PeerState::Disabled { .. }) => {}, + state => panic!("invalid state: {state:?}"), + }; + } + + #[test] + #[cfg(debug_assertions)] + fn peer_disconnected_protocol_accepts() { + let (mut notif, _controller, _notif_service) = development_notifs(); + let peer = PeerId::random(); let set_id = SetId::from(0); + let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), send_back_addr: Multiaddr::empty(), @@ -4079,24 +4476,188 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); - assert!(std::matches!( - notif.incoming[0], - IncomingPeer { alive: true, incoming_id: IncomingIndex(0), .. }, + assert!(notif.incoming.iter().any(|entry| entry.incoming_id == IncomingIndex(0))); + notif.disconnect_peer(&peer, set_id); + + // since the connection was closed, nothing happens for the peer state because + // there is nothing actionable + notif.protocol_report_accept(IncomingIndex(0)); + + match notif.peers.get(&(peer, set_id)) { + Some(&PeerState::Disabled { .. }) => {}, + state => panic!("invalid state: {state:?}"), + }; + + assert!(!notif.incoming.iter().any(|entry| entry.incoming_id == IncomingIndex(0))); + } + + #[test] + #[cfg(debug_assertions)] + fn connection_closed_protocol_accepts() { + let (mut notif, _controller, _notif_service) = development_notifs(); + let peer = PeerId::random(); + let set_id = SetId::from(0); + let conn = ConnectionId::new_unchecked(0); + let connected = ConnectedPoint::Listener { + local_addr: Multiaddr::empty(), + send_back_addr: Multiaddr::empty(), + }; + + notif.on_swarm_event(FromSwarm::ConnectionEstablished( + libp2p::swarm::behaviour::ConnectionEstablished { + peer_id: peer, + connection_id: conn, + endpoint: &connected, + failed_addresses: &[], + other_established: 0usize, + }, + )); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Disabled { .. }))); + + // remote opens a substream, verify that peer state is updated to `Incoming` + notif.on_connection_handler_event( + peer, + conn, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, + ); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); + + notif.on_swarm_event(FromSwarm::ConnectionClosed( + libp2p::swarm::behaviour::ConnectionClosed { + peer_id: peer, + connection_id: ConnectionId::new_unchecked(0), + endpoint: &connected.clone(), + handler: NotifsHandler::new(peer, connected, vec![], None), + remaining_established: 0usize, + }, )); - notif.peers.remove(&(peer, set_id)); - notif.peerset_report_accept(IncomingIndex(0)); + // connection closed, nothing to do + notif.protocol_report_accept(IncomingIndex(0)); + + match notif.peers.get(&(peer, set_id)) { + None => {}, + state => panic!("invalid state: {state:?}"), + }; + } + + #[test] + #[cfg(debug_assertions)] + fn peer_disconnected_protocol_reject() { + let (mut notif, _controller, _notif_service) = development_notifs(); + let peer = PeerId::random(); + let set_id = SetId::from(0); + let conn = ConnectionId::new_unchecked(0); + let connected = ConnectedPoint::Listener { + local_addr: Multiaddr::empty(), + send_back_addr: Multiaddr::empty(), + }; + + notif.on_swarm_event(FromSwarm::ConnectionEstablished( + libp2p::swarm::behaviour::ConnectionEstablished { + peer_id: peer, + connection_id: conn, + endpoint: &connected, + failed_addresses: &[], + other_established: 0usize, + }, + )); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Disabled { .. }))); + + // remote opens a substream, verify that peer state is updated to `Incoming` + notif.on_connection_handler_event( + peer, + conn, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, + ); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); + + assert!(notif.incoming.iter().any(|entry| entry.incoming_id == IncomingIndex(0))); + notif.disconnect_peer(&peer, set_id); + + // since the connection was closed, nothing happens for the peer state because + // there is nothing actionable + notif.protocol_report_reject(IncomingIndex(0)); + + match notif.peers.get(&(peer, set_id)) { + Some(&PeerState::Disabled { .. }) => {}, + state => panic!("invalid state: {state:?}"), + }; + + assert!(!notif.incoming.iter().any(|entry| entry.incoming_id == IncomingIndex(0))); + } + + #[test] + #[cfg(debug_assertions)] + fn connection_closed_protocol_rejects() { + let (mut notif, _controller, _notif_service) = development_notifs(); + let peer = PeerId::random(); + let set_id = SetId::from(0); + let conn = ConnectionId::new_unchecked(0); + let connected = ConnectedPoint::Listener { + local_addr: Multiaddr::empty(), + send_back_addr: Multiaddr::empty(), + }; + + notif.on_swarm_event(FromSwarm::ConnectionEstablished( + libp2p::swarm::behaviour::ConnectionEstablished { + peer_id: peer, + connection_id: conn, + endpoint: &connected, + failed_addresses: &[], + other_established: 0usize, + }, + )); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Disabled { .. }))); + + // remote opens a substream, verify that peer state is updated to `Incoming` + notif.on_connection_handler_event( + peer, + conn, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, + ); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); + + notif.on_swarm_event(FromSwarm::ConnectionClosed( + libp2p::swarm::behaviour::ConnectionClosed { + peer_id: peer, + connection_id: ConnectionId::new_unchecked(0), + endpoint: &connected.clone(), + handler: NotifsHandler::new(peer, connected, vec![], None), + remaining_established: 0usize, + }, + )); + + // connection closed, nothing to do + notif.protocol_report_reject(IncomingIndex(0)); + + match notif.peers.get(&(peer, set_id)) { + None => {}, + state => panic!("invalid state: {state:?}"), + }; } #[test] #[should_panic] #[cfg(debug_assertions)] - fn peerset_report_accept_not_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + fn protocol_report_accept_not_incoming_peer() { + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4121,7 +4682,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4138,14 +4702,14 @@ mod tests { assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); notif.incoming[0].alive = true; - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); } #[test] #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_non_existent_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let endpoint = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), @@ -4157,7 +4721,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(0), endpoint: &endpoint.clone(), - handler: NotifsHandler::new(peer, endpoint, vec![]), + handler: NotifsHandler::new(peer, endpoint, vec![], None), remaining_established: 0usize, }, )); @@ -4165,7 +4729,7 @@ mod tests { #[test] fn disconnect_non_existent_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); @@ -4177,9 +4741,9 @@ mod tests { #[test] fn accept_non_existent_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); - notif.peerset_report_accept(0.into()); + notif.protocol_report_accept(0.into()); assert!(notif.peers.is_empty()); assert!(notif.incoming.is_empty()); @@ -4187,9 +4751,9 @@ mod tests { #[test] fn reject_non_existent_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); - notif.peerset_report_reject(0.into()); + notif.protocol_report_reject(0.into()); assert!(notif.peers.is_empty()); assert!(notif.incoming.is_empty()); @@ -4197,7 +4761,7 @@ mod tests { #[test] fn reject_non_active_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4221,61 +4785,24 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); notif.incoming[0].alive = false; - notif.peerset_report_reject(0.into()); - - assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); - } - - #[test] - #[should_panic] - #[cfg(debug_assertions)] - fn reject_non_existent_peer_but_alive_connection() { - let (mut notif, _controller) = development_notifs(); - let peer = PeerId::random(); - let conn = ConnectionId::new_unchecked(0); - let set_id = SetId::from(0); - let connected = ConnectedPoint::Listener { - local_addr: Multiaddr::empty(), - send_back_addr: Multiaddr::empty(), - }; - - notif.on_swarm_event(FromSwarm::ConnectionEstablished( - libp2p::swarm::behaviour::ConnectionEstablished { - peer_id: peer, - connection_id: conn, - endpoint: &connected, - failed_addresses: &[], - other_established: 0usize, - }, - )); - assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Disabled { .. }))); + notif.protocol_report_reject(0.into()); - // remote opens a substream, verify that peer state is updated to `Incoming` - notif.on_connection_handler_event( - peer, - conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, - ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); - assert!(std::matches!( - notif.incoming[0], - IncomingPeer { alive: true, incoming_id: IncomingIndex(0), .. }, - )); - - notif.peers.remove(&(peer, set_id)); - notif.peerset_report_reject(0.into()); } #[test] #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4299,7 +4826,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4308,7 +4838,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4318,7 +4848,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_disabled_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4343,7 +4873,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4353,7 +4883,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_disabled_pending_enable() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4394,7 +4924,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4404,7 +4934,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_incoming_peer_state_mismatch() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4428,7 +4958,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); notif.incoming[0].alive = false; @@ -4438,7 +4971,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4448,7 +4981,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_enabled_state_mismatch() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4472,7 +5005,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4485,7 +5021,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4495,7 +5031,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_backoff_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4528,7 +5064,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected.clone(), vec![]), + handler: NotifsHandler::new(peer, connected.clone(), vec![], None), remaining_established: 0usize, }, )); @@ -4539,7 +5075,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4549,7 +5085,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn open_result_ok_non_existent_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), diff --git a/substrate/client/network/src/protocol/notifications/handler.rs b/substrate/client/network/src/protocol/notifications/handler.rs index cffdec7d71ee..28662be29fee 100644 --- a/substrate/client/network/src/protocol/notifications/handler.rs +++ b/substrate/client/network/src/protocol/notifications/handler.rs @@ -58,9 +58,12 @@ //! [`NotifsHandlerIn::Open`] has gotten an answer. use crate::{ - protocol::notifications::upgrade::{ - NotificationsIn, NotificationsInSubstream, NotificationsOut, NotificationsOutSubstream, - UpgradeCollec, + protocol::notifications::{ + service::metrics, + upgrade::{ + NotificationsIn, NotificationsInSubstream, NotificationsOut, NotificationsOutSubstream, + UpgradeCollec, + }, }, types::ProtocolName, }; @@ -92,7 +95,7 @@ use std::{ /// Number of pending notifications in asynchronous contexts. /// See [`NotificationsSink::reserve_notification`] for context. -const ASYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 8; +pub(crate) const ASYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 8; /// Number of pending notifications in synchronous contexts. const SYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 2048; @@ -126,11 +129,19 @@ pub struct NotifsHandler { events_queue: VecDeque< ConnectionHandlerEvent, >, + + /// Metrics. + metrics: Option>, } impl NotifsHandler { /// Creates new [`NotifsHandler`]. - pub fn new(peer_id: PeerId, endpoint: ConnectedPoint, protocols: Vec) -> Self { + pub fn new( + peer_id: PeerId, + endpoint: ConnectedPoint, + protocols: Vec, + metrics: Option, + ) -> Self { Self { protocols: protocols .into_iter() @@ -148,6 +159,7 @@ impl NotifsHandler { endpoint, when_connection_open: Instant::now(), events_queue: VecDeque::with_capacity(16), + metrics: metrics.map_or(None, |metrics| Some(Arc::new(metrics))), } } } @@ -303,6 +315,8 @@ pub enum NotifsHandlerOut { OpenDesiredByRemote { /// Index of the protocol in the list of protocols passed at initialization. protocol_index: usize, + /// Received handshake. + handshake: Vec, }, /// The remote would like the substreams to be closed. Send a [`NotifsHandlerIn::Close`] in @@ -331,6 +345,36 @@ pub enum NotifsHandlerOut { #[derive(Debug, Clone)] pub struct NotificationsSink { inner: Arc, + metrics: Option>, +} + +impl NotificationsSink { + /// Create new [`NotificationsSink`]. + /// NOTE: only used for testing but must be `pub` as other crates in `client/network` use this. + pub fn new( + peer_id: PeerId, + ) -> (Self, mpsc::Receiver, mpsc::Receiver) + { + let (async_tx, async_rx) = mpsc::channel(ASYNC_NOTIFICATIONS_BUFFER_SIZE); + let (sync_tx, sync_rx) = mpsc::channel(SYNC_NOTIFICATIONS_BUFFER_SIZE); + ( + NotificationsSink { + inner: Arc::new(NotificationsSinkInner { + peer_id, + async_channel: FuturesMutex::new(async_tx), + sync_channel: Mutex::new(Some(sync_tx)), + }), + metrics: None, + }, + async_rx, + sync_rx, + ) + } + + /// Get reference to metrics. + pub fn metrics(&self) -> &Option> { + &self.metrics + } } #[derive(Debug)] @@ -350,8 +394,8 @@ struct NotificationsSinkInner { /// Message emitted through the [`NotificationsSink`] and processed by the background task /// dedicated to the peer. -#[derive(Debug)] -enum NotificationsSinkMessage { +#[derive(Debug, PartialEq, Eq)] +pub enum NotificationsSinkMessage { /// Message emitted by [`NotificationsSink::reserve_notification`] and /// [`NotificationsSink::write_notification_now`]. Notification { message: Vec }, @@ -379,8 +423,8 @@ impl NotificationsSink { let mut lock = self.inner.sync_channel.lock(); if let Some(tx) = lock.as_mut() { - let result = - tx.try_send(NotificationsSinkMessage::Notification { message: message.into() }); + let message = message.into(); + let result = tx.try_send(NotificationsSinkMessage::Notification { message }); if result.is_err() { // Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the @@ -476,7 +520,10 @@ impl ConnectionHandler for NotifsHandler { match protocol_info.state { State::Closed { pending_opening } => { self.events_queue.push_back(ConnectionHandlerEvent::Custom( - NotifsHandlerOut::OpenDesiredByRemote { protocol_index }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index, + handshake: in_substream_open.handshake, + }, )); protocol_info.state = State::OpenDesiredByRemote { @@ -531,6 +578,7 @@ impl ConnectionHandler for NotifsHandler { async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), }), + metrics: self.metrics.clone(), }; self.protocols[protocol_index].state = State::Open { @@ -881,6 +929,7 @@ pub mod tests { async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), }), + metrics: None, }; let (in_substream, out_substream) = MockSubstream::new(); @@ -1040,6 +1089,7 @@ pub mod tests { }, peer_id: PeerId::random(), events_queue: VecDeque::new(), + metrics: None, } } @@ -1545,6 +1595,7 @@ pub mod tests { async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), }), + metrics: None, }; handler.protocols[0].state = State::Open { @@ -1597,7 +1648,7 @@ pub mod tests { assert!(std::matches!( handler.poll(cx), Poll::Ready(ConnectionHandlerEvent::Custom( - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0, .. }, )) )); assert!(std::matches!( diff --git a/substrate/client/network/src/protocol/notifications/service/metrics.rs b/substrate/client/network/src/protocol/notifications/service/metrics.rs new file mode 100644 index 000000000000..2a57d57c1757 --- /dev/null +++ b/substrate/client/network/src/protocol/notifications/service/metrics.rs @@ -0,0 +1,130 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::types::ProtocolName; + +use prometheus_endpoint::{ + self as prometheus, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, + U64, +}; + +use std::sync::Arc; + +/// Notification metrics. +#[derive(Debug, Clone)] +pub struct Metrics { + // Total number of opened substreams. + pub notifications_streams_opened_total: CounterVec, + + /// Total number of closed substreams. + pub notifications_streams_closed_total: CounterVec, + + /// In/outbound notification sizes. + pub notifications_sizes: HistogramVec, +} + +impl Metrics { + fn register(registry: &Registry) -> Result { + Ok(Self { + notifications_sizes: prometheus::register( + HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "substrate_sub_libp2p_notifications_sizes", + "Sizes of the notifications send to and received from all nodes", + ), + buckets: prometheus::exponential_buckets(64.0, 4.0, 8) + .expect("parameters are always valid values; qed"), + }, + &["direction", "protocol"], + )?, + registry, + )?, + notifications_streams_closed_total: prometheus::register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_notifications_streams_closed_total", + "Total number of notification substreams that have been closed", + ), + &["protocol"], + )?, + registry, + )?, + notifications_streams_opened_total: prometheus::register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_notifications_streams_opened_total", + "Total number of notification substreams that have been opened", + ), + &["protocol"], + )?, + registry, + )?, + }) + } +} + +/// Register metrics. +pub fn register(registry: &Registry) -> Result { + Metrics::register(registry) +} + +/// Register opened substream to Prometheus. +pub fn register_substream_opened(metrics: &Option, protocol: &ProtocolName) { + if let Some(metrics) = metrics { + metrics.notifications_streams_opened_total.with_label_values(&[&protocol]).inc(); + } +} + +/// Register closed substream to Prometheus. +pub fn register_substream_closed(metrics: &Option, protocol: &ProtocolName) { + if let Some(metrics) = metrics { + metrics + .notifications_streams_closed_total + .with_label_values(&[&protocol[..]]) + .inc(); + } +} + +/// Register sent notification to Prometheus. +pub fn register_notification_sent( + metrics: &Option>, + protocol: &ProtocolName, + size: usize, +) { + if let Some(metrics) = metrics { + metrics + .notifications_sizes + .with_label_values(&["out", protocol]) + .observe(size as f64); + } +} + +/// Register received notification to Prometheus. +pub fn register_notification_received( + metrics: &Option, + protocol: &ProtocolName, + size: usize, +) { + if let Some(metrics) = metrics { + metrics + .notifications_sizes + .with_label_values(&["in", protocol]) + .observe(size as f64); + } +} diff --git a/substrate/client/network/src/protocol/notifications/service/mod.rs b/substrate/client/network/src/protocol/notifications/service/mod.rs new file mode 100644 index 000000000000..62e6d88a3d5a --- /dev/null +++ b/substrate/client/network/src/protocol/notifications/service/mod.rs @@ -0,0 +1,634 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Notification service implementation. + +use crate::{ + error, + protocol::notifications::handler::NotificationsSink, + service::traits::{ + Direction, MessageSink, NotificationEvent, NotificationService, ValidationResult, + }, + types::ProtocolName, +}; + +use futures::{ + stream::{FuturesUnordered, Stream}, + StreamExt, +}; +use libp2p::PeerId; +use parking_lot::Mutex; +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::wrappers::ReceiverStream; + +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; + +use std::{collections::HashMap, fmt::Debug, sync::Arc}; + +pub(crate) mod metrics; + +#[cfg(test)] +mod tests; + +/// Logging target for the file. +const LOG_TARGET: &str = "sub-libp2p"; + +/// Default command queue size. +const COMMAND_QUEUE_SIZE: usize = 64; + +/// Type representing subscribers of a notification protocol. +type Subscribers = Arc>>>; + +/// Type represending a distributable message sink. +/// Detached message sink must carry the protocol name for registering metrics. +/// +/// See documentation for [`PeerContext`] for more details. +type NotificationSink = Arc>; + +#[async_trait::async_trait] +impl MessageSink for NotificationSink { + /// Send synchronous `notification` to the peer associated with this [`MessageSink`]. + fn send_sync_notification(&self, notification: Vec) { + let sink = self.lock(); + + metrics::register_notification_sent(&sink.0.metrics(), &sink.1, notification.len()); + sink.0.send_sync_notification(notification); + } + + /// Send an asynchronous `notification` to the peer associated with this [`MessageSink`], + /// allowing sender to exercise backpressure. + /// + /// Returns an error if the peer does not exist. + async fn send_async_notification(&self, notification: Vec) -> Result<(), error::Error> { + // notification sink must be cloned because the lock cannot be held across `.await` + // this makes the implementation less efficient but not prohibitively so as the same + // method is also used by `NetworkService` when sending notifications. + let notification_len = notification.len(); + let sink = self.lock().clone(); + let permit = sink + .0 + .reserve_notification() + .await + .map_err(|_| error::Error::ConnectionClosed)?; + + permit.send(notification).map_err(|_| error::Error::ChannelClosed).map(|res| { + metrics::register_notification_sent(&sink.0.metrics(), &sink.1, notification_len); + res + }) + } +} + +/// Inner notification event to deal with `NotificationsSinks` without exposing that +/// implementation detail to [`NotificationService`] consumers. +#[derive(Debug)] +enum InnerNotificationEvent { + /// Validate inbound substream. + ValidateInboundSubstream { + /// Peer ID. + peer: PeerId, + + /// Received handshake. + handshake: Vec, + + /// `oneshot::Sender` for sending validation result back to `Notifications` + result_tx: oneshot::Sender, + }, + + /// Notification substream open to `peer`. + NotificationStreamOpened { + /// Peer ID. + peer: PeerId, + + /// Direction of the substream. + direction: Direction, + + /// Received handshake. + handshake: Vec, + + /// Negotiated fallback. + negotiated_fallback: Option, + + /// Notification sink. + sink: NotificationsSink, + }, + + /// Substream was closed. + NotificationStreamClosed { + /// Peer ID. + peer: PeerId, + }, + + /// Notification was received from the substream. + NotificationReceived { + /// Peer ID. + peer: PeerId, + + /// Received notification. + notification: Vec, + }, + + /// Notification sink has been replaced. + NotificationSinkReplaced { + /// Peer ID. + peer: PeerId, + + /// Notification sink. + sink: NotificationsSink, + }, +} + +/// Notification commands. +/// +/// Sent by the installed protocols to `Notifications` to open/close/modify substreams. +#[derive(Debug)] +pub enum NotificationCommand { + /// Instruct `Notifications` to open a substream to peer. + #[allow(unused)] + OpenSubstream(PeerId), + + /// Instruct `Notifications` to close the substream to peer. + #[allow(unused)] + CloseSubstream(PeerId), + + /// Set handshake for the notifications protocol. + SetHandshake(Vec), +} + +/// Context assigned to each peer. +/// +/// Contains `NotificationsSink` used by [`NotificationService`] to send notifications +/// and an additional, distributable `NotificationsSink` which the protocol may acquire +/// if it wishes to send notifications through `NotificationsSink` directly. +/// +/// The distributable `NoticationsSink` is wrapped in an `Arc>` to allow +/// `NotificationsService` to swap the underlying sink in case it's replaced. +#[derive(Debug, Clone)] +struct PeerContext { + /// Sink for sending notificaitons. + sink: NotificationsSink, + + /// Distributable notification sink. + shared_sink: NotificationSink, +} + +/// Handle that is passed on to the notifications protocol. +#[derive(Debug)] +pub struct NotificationHandle { + /// Protocol name. + protocol: ProtocolName, + + /// TX channel for sending commands to `Notifications`. + tx: mpsc::Sender, + + /// RX channel for receiving events from `Notifications`. + rx: TracingUnboundedReceiver, + + /// All subscribers of `NotificationEvent`s. + subscribers: Subscribers, + + /// Connected peers. + peers: HashMap, +} + +impl NotificationHandle { + /// Create new [`NotificationHandle`]. + fn new( + protocol: ProtocolName, + tx: mpsc::Sender, + rx: TracingUnboundedReceiver, + subscribers: Arc>>>, + ) -> Self { + Self { protocol, tx, rx, subscribers, peers: HashMap::new() } + } +} + +#[async_trait::async_trait] +impl NotificationService for NotificationHandle { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + todo!("support for opening substreams not implemented yet"); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + todo!("support for closing substreams not implemented yet, call `NetworkService::disconnect_peer()` instead"); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, peer: &PeerId, notification: Vec) { + if let Some(info) = self.peers.get(&peer) { + metrics::register_notification_sent( + &info.sink.metrics(), + &self.protocol, + notification.len(), + ); + + let _ = info.sink.send_sync_notification(notification); + } + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error> { + let notification_len = notification.len(); + let sink = &self.peers.get(&peer).ok_or_else(|| error::Error::PeerDoesntExist(*peer))?.sink; + + sink.reserve_notification() + .await + .map_err(|_| error::Error::ConnectionClosed)? + .send(notification) + .map_err(|_| error::Error::ChannelClosed) + .map(|res| { + metrics::register_notification_sent( + &sink.metrics(), + &self.protocol, + notification_len, + ); + res + }) + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, handshake: Vec) -> Result<(), ()> { + log::trace!(target: LOG_TARGET, "{}: set handshake to {handshake:?}", self.protocol); + + self.tx.send(NotificationCommand::SetHandshake(handshake)).await.map_err(|_| ()) + } + + /// Non-blocking variant of `set_handshake()` that attempts to update the handshake + /// and returns an error if the channel is blocked. + /// + /// Technically the function can return an error if the channel to `Notifications` is closed + /// but that doesn't happen under normal operation. + fn try_set_handshake(&mut self, handshake: Vec) -> Result<(), ()> { + self.tx.try_send(NotificationCommand::SetHandshake(handshake)).map_err(|_| ()) + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + loop { + match self.rx.next().await? { + InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => + return Some(NotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }), + InnerNotificationEvent::NotificationStreamOpened { + peer, + handshake, + negotiated_fallback, + direction, + sink, + } => { + self.peers.insert( + peer, + PeerContext { + sink: sink.clone(), + shared_sink: Arc::new(Mutex::new((sink, self.protocol.clone()))), + }, + ); + return Some(NotificationEvent::NotificationStreamOpened { + peer, + handshake, + direction, + negotiated_fallback, + }) + }, + InnerNotificationEvent::NotificationStreamClosed { peer } => { + self.peers.remove(&peer); + return Some(NotificationEvent::NotificationStreamClosed { peer }) + }, + InnerNotificationEvent::NotificationReceived { peer, notification } => + return Some(NotificationEvent::NotificationReceived { peer, notification }), + InnerNotificationEvent::NotificationSinkReplaced { peer, sink } => { + match self.peers.get_mut(&peer) { + None => log::error!( + "{}: notification sink replaced for {peer} but peer does not exist", + self.protocol + ), + Some(context) => { + context.sink = sink.clone(); + *context.shared_sink.lock() = (sink.clone(), self.protocol.clone()); + }, + } + }, + } + } + } + + // Clone [`NotificationService`] + fn clone(&mut self) -> Result, ()> { + let mut subscribers = self.subscribers.lock(); + let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); + subscribers.push(event_tx); + + Ok(Box::new(NotificationHandle { + protocol: self.protocol.clone(), + tx: self.tx.clone(), + rx: event_rx, + peers: self.peers.clone(), + subscribers: self.subscribers.clone(), + })) + } + + /// Get protocol name. + fn protocol(&self) -> &ProtocolName { + &self.protocol + } + + /// Get message sink of the peer. + fn message_sink(&self, peer: &PeerId) -> Option> { + match self.peers.get(peer) { + Some(context) => Some(Box::new(context.shared_sink.clone())), + None => None, + } + } +} + +/// Channel pair which allows `Notifications` to interact with a protocol. +#[derive(Debug)] +pub struct ProtocolHandlePair { + /// Protocol name. + protocol: ProtocolName, + + /// Subscribers of the notification protocol events. + subscribers: Subscribers, + + // Receiver for notification commands received from the protocol implementation. + rx: mpsc::Receiver, +} + +impl ProtocolHandlePair { + /// Create new [`ProtocolHandlePair`]. + fn new( + protocol: ProtocolName, + subscribers: Subscribers, + rx: mpsc::Receiver, + ) -> Self { + Self { protocol, subscribers, rx } + } + + /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events + /// to the protocol and a stream of commands received from the protocol. + pub(crate) fn split( + self, + ) -> (ProtocolHandle, Box + Send + Unpin>) { + ( + ProtocolHandle::new(self.protocol, self.subscribers), + Box::new(ReceiverStream::new(self.rx)), + ) + } +} + +/// Handle that is passed on to `Notifications` and allows it to directly communicate +/// with the protocol. +#[derive(Debug, Clone)] +pub(crate) struct ProtocolHandle { + /// Protocol name. + protocol: ProtocolName, + + /// Subscribers of the notification protocol. + subscribers: Subscribers, + + /// Number of connected peers. + num_peers: usize, + + /// Delegate validation to `Peerset`. + delegate_to_peerset: bool, + + /// Prometheus metrics. + metrics: Option, +} + +pub(crate) enum ValidationCallResult { + WaitForValidation(oneshot::Receiver), + Delegated, +} + +impl ProtocolHandle { + /// Create new [`ProtocolHandle`]. + fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { + Self { protocol, subscribers, num_peers: 0usize, metrics: None, delegate_to_peerset: false } + } + + /// Set metrics. + pub fn set_metrics(&mut self, metrics: Option) { + self.metrics = metrics; + } + + /// Delegate validation to `Peerset`. + /// + /// Protocols that do not do any validation themselves and only rely on `Peerset` handling + /// validation can disable protocol-side validation entirely by delegating all validation to + /// `Peerset`. + pub fn delegate_to_peerset(&mut self, delegate: bool) { + self.delegate_to_peerset = delegate; + } + + /// Report to the protocol that a substream has been opened and it must be validated by the + /// protocol. + /// + /// Return `oneshot::Receiver` which allows `Notifications` to poll for the validation result + /// from protocol. + pub fn report_incoming_substream( + &self, + peer: PeerId, + handshake: Vec, + ) -> Result { + let subscribers = self.subscribers.lock(); + + log::trace!( + target: LOG_TARGET, + "{}: report incoming substream for {peer}, handshake {handshake:?}", + self.protocol + ); + + if self.delegate_to_peerset { + return Ok(ValidationCallResult::Delegated) + } + + // if there is only one subscriber, `Notifications` can wait directly on the + // `oneshot::channel()`'s RX half without indirection + if subscribers.len() == 1 { + let (result_tx, rx) = oneshot::channel(); + return subscribers[0] + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }) + .map(|_| ValidationCallResult::WaitForValidation(rx)) + .map_err(|_| ()) + } + + // if there are multiple subscribers, create a task which waits for all of the + // validations to finish and returns the combined result to `Notifications` + let mut results: FuturesUnordered<_> = subscribers + .iter() + .filter_map(|subscriber| { + let (result_tx, rx) = oneshot::channel(); + + subscriber + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake: handshake.clone(), + result_tx, + }) + .is_ok() + .then_some(rx) + }) + .collect(); + + let (tx, rx) = oneshot::channel(); + tokio::spawn(async move { + while let Some(event) = results.next().await { + match event { + Err(_) | Ok(ValidationResult::Reject) => + return tx.send(ValidationResult::Reject), + Ok(ValidationResult::Accept) => {}, + } + } + + return tx.send(ValidationResult::Accept) + }); + + Ok(ValidationCallResult::WaitForValidation(rx)) + } + + /// Report to the protocol that a substream has been opened and that it can now use the handle + /// to send notifications to the remote peer. + pub fn report_substream_opened( + &mut self, + peer: PeerId, + direction: Direction, + handshake: Vec, + negotiated_fallback: Option, + sink: NotificationsSink, + ) -> Result<(), ()> { + metrics::register_substream_opened(&self.metrics, &self.protocol); + + let mut subscribers = self.subscribers.lock(); + log::trace!(target: LOG_TARGET, "{}: substream opened for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { + peer, + direction, + handshake: handshake.clone(), + negotiated_fallback: negotiated_fallback.clone(), + sink: sink.clone(), + }) + .is_ok() + }); + self.num_peers += 1; + + Ok(()) + } + + /// Substream was closed. + pub fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { + metrics::register_substream_closed(&self.metrics, &self.protocol); + + let mut subscribers = self.subscribers.lock(); + log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationStreamClosed { peer }) + .is_ok() + }); + self.num_peers -= 1; + + Ok(()) + } + + /// Notification was received from the substream. + pub fn report_notification_received( + &mut self, + peer: PeerId, + notification: Vec, + ) -> Result<(), ()> { + metrics::register_notification_received(&self.metrics, &self.protocol, notification.len()); + + let mut subscribers = self.subscribers.lock(); + log::trace!(target: LOG_TARGET, "{}: notification received from {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationReceived { + peer, + notification: notification.clone(), + }) + .is_ok() + }); + + Ok(()) + } + + /// Notification sink was replaced. + pub fn report_notification_sink_replaced( + &mut self, + peer: PeerId, + sink: NotificationsSink, + ) -> Result<(), ()> { + let mut subscribers = self.subscribers.lock(); + + log::trace!( + target: LOG_TARGET, + "{}: notification sink replaced for {peer:?}", + self.protocol + ); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationSinkReplaced { + peer, + sink: sink.clone(), + }) + .is_ok() + }); + + Ok(()) + } + + /// Get the number of connected peers. + pub fn num_peers(&self) -> usize { + self.num_peers + } +} + +/// Create new (protocol, notification) handle pair. +/// +/// Handle pair allows `Notifications` and the protocol to communicate with each other directly. +pub fn notification_service( + protocol: ProtocolName, +) -> (ProtocolHandlePair, Box) { + let (cmd_tx, cmd_rx) = mpsc::channel(COMMAND_QUEUE_SIZE); + let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); + let subscribers = Arc::new(Mutex::new(vec![event_tx])); + + ( + ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx), + Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers)), + ) +} diff --git a/substrate/client/network/src/protocol/notifications/service/tests.rs b/substrate/client/network/src/protocol/notifications/service/tests.rs new file mode 100644 index 000000000000..02ba9e1711c3 --- /dev/null +++ b/substrate/client/network/src/protocol/notifications/service/tests.rs @@ -0,0 +1,839 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; +use crate::protocol::notifications::handler::{ + NotificationsSinkMessage, ASYNC_NOTIFICATIONS_BUFFER_SIZE, +}; + +use std::future::Future; + +#[tokio::test] +async fn validate_and_accept_substream() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (handle, _stream) = proto.split(); + + let peer_id = PeerId::random(); + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); +} + +#[tokio::test] +async fn substream_opened() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + + let peer_id = PeerId::random(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } +} + +#[tokio::test] +async fn send_sync_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }) + ); +} + +#[tokio::test] +async fn send_async_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }) + ); +} + +#[tokio::test] +async fn send_sync_notification_to_non_existent_peer() { + let (proto, notif) = notification_service("/proto/1".into()); + let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (_handle, _stream) = proto.split(); + let peer = PeerId::random(); + + // as per the original implementation, the call doesn't fail + notif.send_sync_notification(&peer, vec![1, 3, 3, 7]) +} + +#[tokio::test] +async fn send_async_notification_to_non_existent_peer() { + let (proto, notif) = notification_service("/proto/1".into()); + let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (_handle, _stream) = proto.split(); + let peer = PeerId::random(); + + if let Err(error::Error::PeerDoesntExist(peer_id)) = + notif.send_async_notification(&peer, vec![1, 3, 3, 7]).await + { + assert_eq!(peer, peer_id); + } else { + panic!("invalid error received from `send_async_notification()`"); + } +} + +#[tokio::test] +async fn receive_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // notification is received + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); + + if let Some(NotificationEvent::NotificationReceived { peer, notification }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(notification, vec![1, 3, 3, 8]); + } else { + panic!("invalid event received"); + } +} + +#[tokio::test] +async fn backpressure_works() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // fill the message buffer with messages + for i in 0..=ASYNC_NOTIFICATIONS_BUFFER_SIZE { + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, i as u8])) + .is_ready()); + } + + // try to send one more message and verify that the call blocks + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_pending()); + + // release one slot from the buffer for new message + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 0] }) + ); + + // verify that a message can be sent + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_ready()); +} + +#[tokio::test] +async fn peer_disconnects_then_sync_notification_is_sent() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // report that a substream has been closed but don't poll `notif` to receive this + // information + handle.report_substream_closed(peer_id).unwrap(); + drop(sync_rx); + + // as per documentation, error is not reported but the notification is silently dropped + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 7]); +} + +#[tokio::test] +async fn peer_disconnects_then_async_notification_is_sent() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // report that a substream has been closed but don't poll `notif` to receive this + // information + handle.report_substream_closed(peer_id).unwrap(); + drop(async_rx); + + // as per documentation, error is not reported but the notification is silently dropped + if let Err(error::Error::ConnectionClosed) = + notif.send_async_notification(&peer_id, vec![1, 3, 3, 7]).await + { + } else { + panic!("invalid state after calling `send_async_notificatio()` on closed connection") + } +} + +#[tokio::test] +async fn cloned_service_opening_substream_works() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); + let (handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(mut result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + // verify that `notif1` gets the event + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif1.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + // verify that because only one listener has thus far send their result, the result is + // pending + assert!(result_rx.try_recv().is_err()); + + // verify that `notif2` also gets the event + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif2.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); +} + +#[tokio::test] +async fn cloned_service_one_service_rejects_substream() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); + let (handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let mut notif3 = notif2.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(mut result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + for notif in vec![&mut notif1, &mut notif2] { + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + } + + // `notif3` has not yet sent their validation result + assert!(result_rx.try_recv().is_err()); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif3.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Reject).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Reject); +} + +#[tokio::test] +async fn cloned_service_opening_substream_sending_and_receiving_notifications_work() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let mut notif3 = notif1.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + // accept the inbound substream for all services + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that then notification stream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + } + // receive a notification from peer and verify all services receive it + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationReceived { peer, notification }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(notification, vec![1, 3, 3, 8]); + } else { + panic!("invalid event received"); + } + } + + for (i, notif) in vec![&mut notif1, &mut notif2, &mut notif3].iter().enumerate() { + // send notification from each service and verify peer receives it + notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]); + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, i as u8] }) + ); + } + + // close the substream for peer and verify all services receive the event + handle.report_substream_closed(peer_id).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationStreamClosed { peer }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + } else { + panic!("invalid event received"); + } + } +} + +#[tokio::test] +async fn sending_notifications_using_notifications_sink_works() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // get a copy of the notification sink and send a synchronous notification using. + let sink = notif.message_sink(&peer_id).unwrap(); + sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); + + // send notifications using the stored notification sink as well. + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); +} + +#[test] +fn try_to_get_notifications_sink_for_non_existent_peer() { + let (_proto, notif) = notification_service("/proto/1".into()); + assert!(notif.message_sink(&PeerId::random()).is_none()); +} + +#[tokio::test] +async fn notification_sink_replaced() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let ValidationCallResult::WaitForValidation(result_rx) = + handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap() + else { + panic!("peerset not enabled"); + }; + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // get a copy of the notification sink and send a synchronous notification using. + let sink = notif.message_sink(&peer_id).unwrap(); + sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); + + // send notifications using the stored notification sink as well. + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); + + // the initial connection was closed and `Notifications` switched to secondary connection + // and emitted `CustomProtocolReplaced` which informs the local `NotificationService` that + // the notification sink was replaced. + let (new_sink, mut new_async_rx, mut new_sync_rx) = NotificationsSink::new(PeerId::random()); + handle.report_notification_sink_replaced(peer_id, new_sink).unwrap(); + + // drop the old sinks and poll `notif` once to register the sink replacement + drop(sync_rx); + drop(async_rx); + + futures::future::poll_fn(|cx| { + let _ = std::pin::Pin::new(&mut notif.next_event()).poll(cx); + std::task::Poll::Ready(()) + }) + .await; + + // verify that using the `NotificationService` API automatically results in using the correct + // sink + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + new_sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + new_async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); + + // now send two notifications using the acquired message sink and verify that + // it's also updated + sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + + assert_eq!( + new_sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + new_async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); +} + +#[tokio::test] +async fn set_handshake() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (_handle, mut stream) = proto.split(); + + assert!(notif.try_set_handshake(vec![1, 3, 3, 7]).is_ok()); + + match stream.next().await { + Some(NotificationCommand::SetHandshake(handshake)) => { + assert_eq!(handshake, vec![1, 3, 3, 7]); + }, + _ => panic!("invalid event received"), + } + + for _ in 0..COMMAND_QUEUE_SIZE { + assert!(notif.try_set_handshake(vec![1, 3, 3, 7]).is_ok()); + } + + assert!(notif.try_set_handshake(vec![1, 3, 3, 7]).is_err()); +} diff --git a/substrate/client/network/src/protocol/notifications/tests.rs b/substrate/client/network/src/protocol/notifications/tests.rs index d57c24144f57..92d269f89c31 100644 --- a/substrate/client/network/src/protocol/notifications/tests.rs +++ b/substrate/client/network/src/protocol/notifications/tests.rs @@ -22,6 +22,7 @@ use crate::{ peer_store::PeerStore, protocol::notifications::{Notifications, NotificationsOut, ProtocolConfig}, protocol_controller::{ProtoSetConfig, ProtocolController, SetId}, + service::traits::{NotificationEvent, ValidationResult}, }; use futures::{future::BoxFuture, prelude::*}; @@ -70,6 +71,8 @@ fn build_nodes() -> (Swarm, Swarm) { .timeout(Duration::from_secs(20)) .boxed(); + let (protocol_handle_pair, mut notif_service) = + crate::protocol::notifications::service::notification_service("/foo".into()); let peer_store = PeerStore::new(if index == 0 { keypairs.iter().skip(1).map(|keypair| keypair.public().to_peer_id()).collect() } else { @@ -91,16 +94,22 @@ fn build_nodes() -> (Swarm, Swarm) { Box::new(peer_store.handle()), ); + let (notif_handle, command_stream) = protocol_handle_pair.split(); let behaviour = CustomProtoWithAddr { inner: Notifications::new( vec![controller_handle], from_controller, - iter::once(ProtocolConfig { - name: "/foo".into(), - fallback_names: Vec::new(), - handshake: Vec::new(), - max_notification_size: 1024 * 1024, - }), + &None, + iter::once(( + ProtocolConfig { + name: "/foo".into(), + fallback_names: Vec::new(), + handshake: Vec::new(), + max_notification_size: 1024 * 1024, + }, + notif_handle, + command_stream, + )), ), peer_store_future: peer_store.run().boxed(), protocol_controller_future: controller.run().boxed(), @@ -118,6 +127,16 @@ fn build_nodes() -> (Swarm, Swarm) { }; let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.spawn(async move { + loop { + if let NotificationEvent::ValidateInboundSubstream { result_tx, .. } = + notif_service.next_event().await.unwrap() + { + result_tx.send(ValidationResult::Accept).unwrap(); + } + } + }); + let mut swarm = SwarmBuilder::with_executor( transport, behaviour, diff --git a/substrate/client/network/src/protocol_controller.rs b/substrate/client/network/src/protocol_controller.rs index 3a305011ded0..4c8f119baa20 100644 --- a/substrate/client/network/src/protocol_controller.rs +++ b/substrate/client/network/src/protocol_controller.rs @@ -847,6 +847,7 @@ mod tests { use super::*; use crate::{peer_store::PeerStoreProvider, ReputationChange}; use libp2p::PeerId; + use sc_network_common::role::ObservedRole; use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; use std::collections::HashSet; @@ -858,8 +859,10 @@ mod tests { fn is_banned(&self, peer_id: &PeerId) -> bool; fn register_protocol(&self, protocol_handle: ProtocolHandle); fn report_disconnect(&mut self, peer_id: PeerId); + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole); fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); fn peer_reputation(&self, peer_id: &PeerId) -> i32; + fn peer_role(&self, peer_id: &PeerId) -> Option; fn outgoing_candidates<'a>(&self, count: usize, ignored: HashSet<&'a PeerId>) -> Vec; } } diff --git a/substrate/client/network/src/service.rs b/substrate/client/network/src/service.rs index c1df48ad7858..43a3ab09115c 100644 --- a/substrate/client/network/src/service.rs +++ b/substrate/client/network/src/service.rs @@ -54,6 +54,7 @@ use crate::{ ReputationChange, }; +use codec::DecodeAll; use either::Either; use futures::{channel::oneshot, prelude::*}; #[allow(deprecated)] @@ -71,10 +72,13 @@ use libp2p::{ Multiaddr, PeerId, }; use log::{debug, error, info, trace, warn}; -use metrics::{Histogram, HistogramVec, MetricSources, Metrics}; +use metrics::{Histogram, MetricSources, Metrics}; use parking_lot::Mutex; -use sc_network_common::ExHashT; +use sc_network_common::{ + role::{ObservedRole, Roles}, + ExHashT, +}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::Block as BlockT; @@ -118,12 +122,6 @@ pub struct NetworkService { bandwidth: Arc, /// Channel that sends messages to the actual worker. to_worker: TracingUnboundedSender, - /// For each peer and protocol combination, an object that allows sending notifications to - /// that peer. Updated by the [`NetworkWorker`]. - peers_notifications_sinks: Arc>>, - /// Field extracted from the [`Metrics`] struct and necessary to report the - /// notifications-related metrics. - notifications_sizes_metric: Option, /// Protocol name -> `SetId` mapping for notification protocols. The map never changes after /// initialization. notification_protocol_ids: HashMap, @@ -132,6 +130,8 @@ pub struct NetworkService { protocol_handles: Vec, /// Shortcut to sync protocol handle (`protocol_handles[0]`). sync_protocol_handle: protocol_controller::ProtocolHandle, + /// Handle to `PeerStore`. + peer_store_handle: PeerStoreHandle, /// Marker to pin the `H` generic. Serves no purpose except to not break backwards /// compatibility. _marker: PhantomData, @@ -199,7 +199,7 @@ where )?; for notification_protocol in ¬ification_protocols { ensure_addresses_consistent_with_transport( - notification_protocol.set_config.reserved_nodes.iter().map(|x| &x.multiaddr), + notification_protocol.set_config().reserved_nodes.iter().map(|x| &x.multiaddr), &network_config.transport, )?; } @@ -241,7 +241,7 @@ where .map(|cfg| usize::try_from(cfg.max_response_size).unwrap_or(usize::MAX)); let notifs_max = notification_protocols .iter() - .map(|cfg| usize::try_from(cfg.max_notification_size).unwrap_or(usize::MAX)); + .map(|cfg| usize::try_from(cfg.max_notification_size()).unwrap_or(usize::MAX)); // A "default" max is added to cover all the other protocols: ping, identify, // kademlia, block announces, and transactions. @@ -273,7 +273,7 @@ where // We must prepend a hardcoded default peer set to notification protocols. let all_peer_sets_iter = iter::once(&network_config.default_peers_set) - .chain(notification_protocols.iter().map(|protocol| &protocol.set_config)); + .chain(notification_protocols.iter().map(|protocol| protocol.set_config())); let (protocol_handles, protocol_controllers): (Vec<_>, Vec<_>) = all_peer_sets_iter .enumerate() @@ -312,21 +312,9 @@ where iter::once(¶ms.block_announce_config) .chain(notification_protocols.iter()) .enumerate() - .map(|(index, protocol)| { - (protocol.notifications_protocol.clone(), SetId::from(index)) - }) + .map(|(index, protocol)| (protocol.protocol_name().clone(), SetId::from(index))) .collect(); - let protocol = Protocol::new( - From::from(¶ms.role), - notification_protocols.clone(), - params.block_announce_config, - params.peer_store.clone(), - protocol_handles.clone(), - from_protocol_controllers, - params.tx, - )?; - let known_addresses = { // Collect all reserved nodes and bootnodes addresses. let mut addresses: Vec<_> = network_config @@ -336,7 +324,7 @@ where .map(|reserved| (reserved.peer_id, reserved.multiaddr.clone())) .chain(notification_protocols.iter().flat_map(|protocol| { protocol - .set_config + .set_config() .reserved_nodes .iter() .map(|reserved| (reserved.peer_id, reserved.multiaddr.clone())) @@ -389,6 +377,16 @@ where let num_connected = Arc::new(AtomicUsize::new(0)); let external_addresses = Arc::new(Mutex::new(HashSet::new())); + let (protocol, notif_protocol_handles) = Protocol::new( + From::from(¶ms.role), + ¶ms.metrics_registry, + notification_protocols, + params.block_announce_config, + params.peer_store.clone(), + protocol_handles.clone(), + from_protocol_controllers, + )?; + // Build the swarm. let (mut swarm, bandwidth): (Swarm>, _) = { let user_agent = @@ -508,7 +506,6 @@ where } let listen_addresses = Arc::new(Mutex::new(HashSet::new())); - let peers_notifications_sinks = Arc::new(Mutex::new(HashMap::new())); let service = Arc::new(NetworkService { bandwidth, @@ -518,13 +515,10 @@ where local_peer_id, local_identity, to_worker, - peers_notifications_sinks: peers_notifications_sinks.clone(), - notifications_sizes_metric: metrics - .as_ref() - .map(|metrics| metrics.notifications_sizes.clone()), notification_protocol_ids, protocol_handles, sync_protocol_handle, + peer_store_handle: params.peer_store.clone(), _marker: PhantomData, _block: Default::default(), }); @@ -539,8 +533,8 @@ where metrics, boot_node_ids, reported_invalid_boot_nodes: Default::default(), - peers_notifications_sinks, peer_store_handle: params.peer_store, + notif_protocol_handles, _marker: Default::default(), _block: Default::default(), }) @@ -567,7 +561,7 @@ where /// Returns the number of peers we're connected to. pub fn num_connected_peers(&self) -> usize { - self.network_service.behaviour().user_protocol().num_connected_peers() + self.network_service.behaviour().user_protocol().num_sync_peers() } /// Adds an address for a node. @@ -991,6 +985,16 @@ where fn sync_num_connected(&self) -> usize { self.num_connected.load(Ordering::Relaxed) } + + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option { + match Roles::decode_all(&mut &handshake[..]) { + Ok(role) => Some(role.into()), + Err(_) => { + log::debug!(target: "sub-libp2p", "handshake doesn't contain peer role: {handshake:?}"); + self.peer_store_handle.peer_role(&peer_id) + }, + } + } } impl NetworkEventStream for NetworkService @@ -1010,68 +1014,20 @@ where B: BlockT + 'static, H: ExHashT, { - fn write_notification(&self, target: PeerId, protocol: ProtocolName, message: Vec) { - // We clone the `NotificationsSink` in order to be able to unlock the network-wide - // `peers_notifications_sinks` mutex as soon as possible. - let sink = { - let peers_notifications_sinks = self.peers_notifications_sinks.lock(); - if let Some(sink) = peers_notifications_sinks.get(&(target, protocol.clone())) { - sink.clone() - } else { - // Notification silently discarded, as documented. - debug!( - target: "sub-libp2p", - "Attempted to send notification on missing or closed substream: {}, {:?}", - target, protocol, - ); - return - } - }; - - if let Some(notifications_sizes_metric) = self.notifications_sizes_metric.as_ref() { - notifications_sizes_metric - .with_label_values(&["out", &protocol]) - .observe(message.len() as f64); - } - - // Sending is communicated to the `NotificationsSink`. - trace!( - target: "sub-libp2p", - "External API => Notification({:?}, {:?}, {} bytes)", - target, protocol, message.len() - ); - trace!(target: "sub-libp2p", "Handler({:?}) <= Sync notification", target); - sink.send_sync_notification(message); + fn write_notification(&self, _target: PeerId, _protocol: ProtocolName, _message: Vec) { + unimplemented!(); } fn notification_sender( &self, - target: PeerId, - protocol: ProtocolName, + _target: PeerId, + _protocol: ProtocolName, ) -> Result, NotificationSenderError> { - // We clone the `NotificationsSink` in order to be able to unlock the network-wide - // `peers_notifications_sinks` mutex as soon as possible. - let sink = { - let peers_notifications_sinks = self.peers_notifications_sinks.lock(); - if let Some(sink) = peers_notifications_sinks.get(&(target, protocol.clone())) { - sink.clone() - } else { - return Err(NotificationSenderError::Closed) - } - }; - - let notification_size_metric = self - .notifications_sizes_metric - .as_ref() - .map(|histogram| histogram.with_label_values(&["out", &protocol])); - - Ok(Box::new(NotificationSender { sink, protocol_name: protocol, notification_size_metric })) + unimplemented!(); } - fn set_notification_handshake(&self, protocol: ProtocolName, handshake: Vec) { - let _ = self - .to_worker - .unbounded_send(ServiceToWorkerMsg::SetNotificationHandshake(protocol, handshake)); + fn set_notification_handshake(&self, _protocol: ProtocolName, _handshake: Vec) { + unimplemented!(); } } @@ -1209,7 +1165,6 @@ enum ServiceToWorkerMsg { pending_response: oneshot::Sender>, }, DisconnectPeer(PeerId, ProtocolName), - SetNotificationHandshake(ProtocolName, Vec), } /// Main network worker. Must be polled in order for the network to advance. @@ -1239,11 +1194,10 @@ where boot_node_ids: Arc>>, /// Boot nodes that we already have reported as invalid. reported_invalid_boot_nodes: HashSet, - /// For each peer and protocol combination, an object that allows sending notifications to - /// that peer. Shared with the [`NetworkService`]. - peers_notifications_sinks: Arc>>, /// Peer reputation store handle. peer_store_handle: PeerStoreHandle, + /// Notification protocol handles. + notif_protocol_handles: Vec, /// Marker to pin the `H` generic. Serves no purpose except to not break backwards /// compatibility. _marker: PhantomData, @@ -1282,8 +1236,7 @@ where }; // Update the `num_connected` count shared with the `NetworkService`. - let num_connected_peers = - self.network_service.behaviour_mut().user_protocol_mut().num_connected_peers(); + let num_connected_peers = self.network_service.behaviour().user_protocol().num_sync_peers(); self.num_connected.store(num_connected_peers, Ordering::Relaxed); if let Some(metrics) = self.metrics.as_ref() { @@ -1353,11 +1306,6 @@ where .behaviour_mut() .user_protocol_mut() .disconnect_peer(&who, protocol_name), - ServiceToWorkerMsg::SetNotificationHandshake(protocol, handshake) => self - .network_service - .behaviour_mut() - .user_protocol_mut() - .set_notification_handshake(protocol, handshake), } } @@ -1472,47 +1420,27 @@ where }, SwarmEvent::Behaviour(BehaviourOut::NotificationStreamOpened { remote, - protocol, + set_id, + direction, negotiated_fallback, notifications_sink, - role, received_handshake, }) => { - if let Some(metrics) = self.metrics.as_ref() { - metrics - .notifications_streams_opened_total - .with_label_values(&[&protocol]) - .inc(); - } - { - let mut peers_notifications_sinks = self.peers_notifications_sinks.lock(); - let _previous_value = peers_notifications_sinks - .insert((remote, protocol.clone()), notifications_sink); - debug_assert!(_previous_value.is_none()); - } - self.event_streams.send(Event::NotificationStreamOpened { + let _ = self.notif_protocol_handles[usize::from(set_id)].report_substream_opened( remote, - protocol, - negotiated_fallback, - role, + direction, received_handshake, - }); + negotiated_fallback, + notifications_sink, + ); }, SwarmEvent::Behaviour(BehaviourOut::NotificationStreamReplaced { remote, - protocol, + set_id, notifications_sink, }) => { - let mut peers_notifications_sinks = self.peers_notifications_sinks.lock(); - if let Some(s) = peers_notifications_sinks.get_mut(&(remote, protocol)) { - *s = notifications_sink; - } else { - error!( - target: "sub-libp2p", - "NotificationStreamReplaced for non-existing substream" - ); - debug_assert!(false); - } + let _ = self.notif_protocol_handles[usize::from(set_id)] + .report_notification_sink_replaced(remote, notifications_sink); // TODO: Notifications might have been lost as a result of the previous // connection being dropped, and as a result it would be preferable to notify @@ -1535,31 +1463,17 @@ where // role, // }); }, - SwarmEvent::Behaviour(BehaviourOut::NotificationStreamClosed { remote, protocol }) => { - if let Some(metrics) = self.metrics.as_ref() { - metrics - .notifications_streams_closed_total - .with_label_values(&[&protocol[..]]) - .inc(); - } - self.event_streams - .send(Event::NotificationStreamClosed { remote, protocol: protocol.clone() }); - { - let mut peers_notifications_sinks = self.peers_notifications_sinks.lock(); - let _previous_value = peers_notifications_sinks.remove(&(remote, protocol)); - debug_assert!(_previous_value.is_some()); - } + SwarmEvent::Behaviour(BehaviourOut::NotificationStreamClosed { remote, set_id }) => { + let _ = self.notif_protocol_handles[usize::from(set_id)] + .report_substream_closed(remote); }, - SwarmEvent::Behaviour(BehaviourOut::NotificationsReceived { remote, messages }) => { - if let Some(metrics) = self.metrics.as_ref() { - for (protocol, message) in &messages { - metrics - .notifications_sizes - .with_label_values(&["in", protocol]) - .observe(message.len() as f64); - } - } - self.event_streams.send(Event::NotificationsReceived { remote, messages }); + SwarmEvent::Behaviour(BehaviourOut::NotificationsReceived { + remote, + set_id, + notification, + }) => { + let _ = self.notif_protocol_handles[usize::from(set_id)] + .report_notification_received(remote, notification); }, SwarmEvent::Behaviour(BehaviourOut::Dht(event, duration)) => { if let Some(metrics) = self.metrics.as_ref() { diff --git a/substrate/client/network/src/service/metrics.rs b/substrate/client/network/src/service/metrics.rs index 13bc4b4e7aff..c349fd98c76b 100644 --- a/substrate/client/network/src/service/metrics.rs +++ b/substrate/client/network/src/service/metrics.rs @@ -61,9 +61,6 @@ pub struct Metrics { pub kbuckets_num_nodes: GaugeVec, pub listeners_local_addresses: Gauge, pub listeners_errors_total: Counter, - pub notifications_sizes: HistogramVec, - pub notifications_streams_closed_total: CounterVec, - pub notifications_streams_opened_total: CounterVec, pub peerset_num_discovered: Gauge, pub pending_connections: Gauge, pub pending_connections_errors_total: CounterVec, @@ -153,31 +150,6 @@ impl Metrics { "substrate_sub_libp2p_listeners_errors_total", "Total number of non-fatal errors reported by a listener" )?, registry)?, - notifications_sizes: prometheus::register(HistogramVec::new( - HistogramOpts { - common_opts: Opts::new( - "substrate_sub_libp2p_notifications_sizes", - "Sizes of the notifications send to and received from all nodes" - ), - buckets: prometheus::exponential_buckets(64.0, 4.0, 8) - .expect("parameters are always valid values; qed"), - }, - &["direction", "protocol"] - )?, registry)?, - notifications_streams_closed_total: prometheus::register(CounterVec::new( - Opts::new( - "substrate_sub_libp2p_notifications_streams_closed_total", - "Total number of notification substreams that have been closed" - ), - &["protocol"] - )?, registry)?, - notifications_streams_opened_total: prometheus::register(CounterVec::new( - Opts::new( - "substrate_sub_libp2p_notifications_streams_opened_total", - "Total number of notification substreams that have been opened" - ), - &["protocol"] - )?, registry)?, peerset_num_discovered: prometheus::register(Gauge::new( "substrate_sub_libp2p_peerset_num_discovered", "Number of nodes stored in the peerset manager", diff --git a/substrate/client/network/src/service/signature.rs b/substrate/client/network/src/service/signature.rs index 024f60e4c466..5b2ba6be8cf8 100644 --- a/substrate/client/network/src/service/signature.rs +++ b/substrate/client/network/src/service/signature.rs @@ -18,6 +18,8 @@ // // If you read this, you are very thorough, congratulations. +//! Signature-related code + use libp2p::{ identity::{Keypair, PublicKey}, PeerId, diff --git a/substrate/client/network/src/service/traits.rs b/substrate/client/network/src/service/traits.rs index bed325ede4a8..f66e810be11a 100644 --- a/substrate/client/network/src/service/traits.rs +++ b/substrate/client/network/src/service/traits.rs @@ -18,8 +18,11 @@ // // If you read this, you are very thorough, congratulations. +//! Traits defined by `sc-network`. + use crate::{ config::MultiaddrWithPeerId, + error, event::Event, request_responses::{IfDisconnected, RequestFailure}, service::signature::Signature, @@ -30,7 +33,9 @@ use crate::{ use futures::{channel::oneshot, Stream}; use libp2p::{Multiaddr, PeerId}; -use std::{collections::HashSet, future::Future, pin::Pin, sync::Arc}; +use sc_network_common::role::ObservedRole; + +use std::{collections::HashSet, fmt::Debug, future::Future, pin::Pin, sync::Arc}; pub use libp2p::{identity::SigningError, kad::record::Key as KademliaKey}; @@ -221,6 +226,14 @@ pub trait NetworkPeers { /// Returns the number of peers in the sync peer set we're connected to. fn sync_num_connected(&self) -> usize; + + /// Attempt to get peer role. + /// + /// Right now the peer role is decoded from the received handshake for all protocols + /// (`/block-announces/1` has other information as well). If the handshake cannot be + /// decoded into a role, the role queried from `PeerStore` and if the role is not stored + /// there either, `None` is returned and the peer should be discarded. + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option; } // Manual implementation to avoid extra boxing here @@ -296,6 +309,10 @@ where fn sync_num_connected(&self) -> usize { T::sync_num_connected(self) } + + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option { + T::peer_role(self, peer_id, handshake) + } } /// Provides access to network-level event stream. @@ -611,3 +628,189 @@ where T::new_best_block_imported(self, hash, number) } } + +/// Substream acceptance result. +#[derive(Debug, PartialEq, Eq)] +pub enum ValidationResult { + /// Accept inbound substream. + Accept, + + /// Reject inbound substream. + Reject, +} + +/// Substream direction. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum Direction { + /// Substream opened by the remote node. + Inbound, + + /// Substream opened by the local node. + Outbound, +} + +impl Direction { + /// Is the direction inbound. + pub fn is_inbound(&self) -> bool { + std::matches!(self, Direction::Inbound) + } +} + +/// Events received by the protocol from `Notifications`. +#[derive(Debug)] +pub enum NotificationEvent { + /// Validate inbound substream. + ValidateInboundSubstream { + /// Peer ID. + peer: PeerId, + + /// Received handshake. + handshake: Vec, + + /// `oneshot::Sender` for sending validation result back to `Notifications` + result_tx: tokio::sync::oneshot::Sender, + }, + + /// Remote identified by `PeerId` opened a substream and sent `Handshake`. + /// Validate `Handshake` and report status (accept/reject) to `Notifications`. + NotificationStreamOpened { + /// Peer ID. + peer: PeerId, + + /// Is the substream inbound or outbound. + direction: Direction, + + /// Received handshake. + handshake: Vec, + + /// Negotiated fallback. + negotiated_fallback: Option, + }, + + /// Substream was closed. + NotificationStreamClosed { + /// Peer Id. + peer: PeerId, + }, + + /// Notification was received from the substream. + NotificationReceived { + /// Peer ID. + peer: PeerId, + + /// Received notification. + notification: Vec, + }, +} + +/// Notification service +/// +/// Defines behaviors that both the protocol implementations and `Notifications` can expect from +/// each other. +/// +/// `Notifications` can send two different kinds of information to protocol: +/// * substream-related information +/// * notification-related information +/// +/// When an unvalidated, inbound substream is received by `Notifications`, it sends the inbound +/// stream information (peer ID, handshake) to protocol for validation. Protocol must then verify +/// that the handshake is valid (and in the future that it has a slot it can allocate for the peer) +/// and then report back the `ValidationResult` which is either `Accept` or `Reject`. +/// +/// After the validation result has been received by `Notifications`, it prepares the +/// substream for communication by initializing the necessary sinks and emits +/// `NotificationStreamOpened` which informs the protocol that the remote peer is ready to receive +/// notifications. +/// +/// Two different flavors of sending options are provided: +/// * synchronous sending ([`NotificationService::send_sync_notification()`]) +/// * asynchronous sending ([`NotificationService::send_async_notification()`]) +/// +/// The former is used by the protocols not ready to exercise backpressure and the latter by the +/// protocols that can do it. +/// +/// Both local and remote peer can close the substream at any time. Local peer can do so by calling +/// [`NotificationService::close_substream()`] which instructs `Notifications` to close the +/// substream. Remote closing the substream is indicated to the local peer by receiving +/// [`NotificationEvent::NotificationStreamClosed`] event. +/// +/// In case the protocol must update its handshake while it's operating (such as updating the best +/// block information), it can do so by calling [`NotificationService::set_handshake()`] +/// which instructs `Notifications` to update the handshake it stored during protocol +/// initialization. +/// +/// All peer events are multiplexed on the same incoming event stream from `Notifications` and thus +/// each event carries a `PeerId` so the protocol knows whose information to update when receiving +/// an event. +#[async_trait::async_trait] +pub trait NotificationService: Debug + Send { + /// Instruct `Notifications` to open a new substream for `peer`. + /// + /// `dial_if_disconnected` informs `Notifications` whether to dial + // the peer if there is currently no active connection to it. + // + // NOTE: not offered by the current implementation + async fn open_substream(&mut self, peer: PeerId) -> Result<(), ()>; + + /// Instruct `Notifications` to close substream for `peer`. + // + // NOTE: not offered by the current implementation + async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()>; + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, peer: &PeerId, notification: Vec); + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + /// + /// Returns an error if the peer doesn't exist. + async fn send_async_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error>; + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, handshake: Vec) -> Result<(), ()>; + + /// Non-blocking variant of `set_handshake()` that attempts to update the handshake + /// and returns an error if the channel is blocked. + /// + /// Technically the function can return an error if the channel to `Notifications` is closed + /// but that doesn't happen under normal operation. + fn try_set_handshake(&mut self, handshake: Vec) -> Result<(), ()>; + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option; + + /// Make a copy of the object so it can be shared between protocol components + /// who wish to have access to the same underlying notification protocol. + fn clone(&mut self) -> Result, ()>; + + /// Get protocol name of the `NotificationService`. + fn protocol(&self) -> &ProtocolName; + + /// Get message sink of the peer. + fn message_sink(&self, peer: &PeerId) -> Option>; +} + +/// Message sink for peers. +/// +/// If protocol cannot use [`NotificationService`] to send notifications to peers and requires, +/// e.g., notifications to be sent in another task, the protocol may acquire a [`MessageSink`] +/// object for each peer by calling [`NotificationService::message_sink()`]. Calling this +/// function returns an object which allows the protocol to send notifications to the remote peer. +/// +/// Use of this API is discouraged as it's not as performant as sending notifications through +/// [`NotificationService`] due to synchronization required to keep the underlying notification +/// sink up to date with possible sink replacement events. +#[async_trait::async_trait] +pub trait MessageSink: Send + Sync { + /// Send synchronous `notification` to the peer associated with this [`MessageSink`]. + fn send_sync_notification(&self, notification: Vec); + + /// Send an asynchronous `notification` to to the peer associated with this [`MessageSink`], + /// allowing sender to exercise backpressure. + /// + /// Returns an error if the peer does not exist. + async fn send_async_notification(&self, notification: Vec) -> Result<(), error::Error>; +} diff --git a/substrate/client/network/statement/src/lib.rs b/substrate/client/network/statement/src/lib.rs index 69d4faa13ef2..5187e681d83c 100644 --- a/substrate/client/network/statement/src/lib.rs +++ b/substrate/client/network/statement/src/lib.rs @@ -21,12 +21,13 @@ //! Usage: //! //! - Use [`StatementHandlerPrototype::new`] to create a prototype. -//! - Pass the return value of [`StatementHandlerPrototype::set_config`] to the network -//! configuration as an extra peers set. +//! - Pass the `NonDefaultSetConfig` returned from [`StatementHandlerPrototype::new`] to the network +//! configuration as an extra peers set. //! - Use [`StatementHandlerPrototype::build`] then [`StatementHandler::run`] to obtain a //! `Future` that processes statements. use crate::config::*; + use codec::{Decode, Encode}; use futures::{channel::oneshot, prelude::*, stream::FuturesUnordered, FutureExt}; use libp2p::{multiaddr, PeerId}; @@ -34,7 +35,7 @@ use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; use sc_network::{ config::{NonDefaultSetConfig, NonReservedPeerMode, SetConfig}, error, - event::Event, + service::traits::{NotificationEvent, NotificationService, ValidationResult}, types::ProtocolName, utils::{interval, LruHashSet}, NetworkEventStream, NetworkNotification, NetworkPeers, @@ -101,35 +102,35 @@ impl Metrics { /// Prototype for a [`StatementHandler`]. pub struct StatementHandlerPrototype { protocol_name: ProtocolName, + notification_service: Box, } impl StatementHandlerPrototype { /// Create a new instance. - pub fn new>(genesis_hash: Hash, fork_id: Option<&str>) -> Self { + pub fn new>( + genesis_hash: Hash, + fork_id: Option<&str>, + ) -> (Self, NonDefaultSetConfig) { let genesis_hash = genesis_hash.as_ref(); let protocol_name = if let Some(fork_id) = fork_id { format!("/{}/{}/statement/1", array_bytes::bytes2hex("", genesis_hash), fork_id) } else { format!("/{}/statement/1", array_bytes::bytes2hex("", genesis_hash)) }; - - Self { protocol_name: protocol_name.into() } - } - - /// Returns the configuration of the set to put in the network configuration. - pub fn set_config(&self) -> NonDefaultSetConfig { - NonDefaultSetConfig { - notifications_protocol: self.protocol_name.clone(), - fallback_names: Vec::new(), - max_notification_size: MAX_STATEMENT_SIZE, - handshake: None, - set_config: SetConfig { + let (config, notification_service) = NonDefaultSetConfig::new( + protocol_name.clone().into(), + Vec::new(), + MAX_STATEMENT_SIZE, + None, + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ); + + (Self { protocol_name: protocol_name.into(), notification_service }, config) } /// Turns the prototype into the actual handler. @@ -147,7 +148,6 @@ impl StatementHandlerPrototype { metrics_registry: Option<&Registry>, executor: impl Fn(Pin + Send>>) + Send, ) -> error::Result> { - let net_event_stream = network.event_stream("statement-handler-net"); let sync_event_stream = sync.event_stream("statement-handler-sync"); let (queue_sender, mut queue_receiver) = async_channel::bounded(100_000); @@ -176,6 +176,7 @@ impl StatementHandlerPrototype { let handler = StatementHandler { protocol_name: self.protocol_name, + notification_service: self.notification_service, propagate_timeout: (Box::pin(interval(PROPAGATE_TIMEOUT)) as Pin + Send>>) .fuse(), @@ -183,7 +184,6 @@ impl StatementHandlerPrototype { pending_statements_peers: HashMap::new(), network, sync, - net_event_stream: net_event_stream.fuse(), sync_event_stream: sync_event_stream.fuse(), peers: HashMap::new(), statement_store, @@ -219,10 +219,10 @@ pub struct StatementHandler< network: N, /// Syncing service. sync: S, - /// Stream of networking events. - net_event_stream: stream::Fuse + Send>>>, /// Receiver for syncing-related events. sync_event_stream: stream::Fuse + Send>>>, + /// Notification service. + notification_service: Box, // All connected peers peers: HashMap, statement_store: Arc, @@ -261,14 +261,6 @@ where log::warn!(target: LOG_TARGET, "Inconsistent state, no peers for pending statement!"); } }, - network_event = self.net_event_stream.next() => { - if let Some(network_event) = network_event { - self.handle_network_event(network_event).await; - } else { - // Networking has seemingly closed. Closing as well. - return; - } - }, sync_event = self.sync_event_stream.next() => { if let Some(sync_event) = sync_event { self.handle_sync_event(sync_event); @@ -277,6 +269,14 @@ where return; } } + event = self.notification_service.next_event().fuse() => { + if let Some(event) = event { + self.handle_notification_event(event) + } else { + // `Notifications` has seemingly closed. Closing as well. + return + } + } } } } @@ -306,14 +306,24 @@ where } } - async fn handle_network_event(&mut self, event: Event) { + fn handle_notification_event(&mut self, event: NotificationEvent) { match event { - Event::Dht(_) => {}, - Event::NotificationStreamOpened { remote, protocol, role, .. } - if protocol == self.protocol_name => - { + NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx, .. } => { + // only accept peers whose role can be determined + let result = self + .network + .peer_role(peer, handshake) + .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); + let _ = result_tx.send(result); + }, + NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { + let Some(role) = self.network.peer_role(peer, handshake) else { + log::debug!(target: LOG_TARGET, "role for {peer} couldn't be determined"); + return + }; + let _was_in = self.peers.insert( - remote, + peer, Peer { known_statements: LruHashSet::new( NonZeroUsize::new(MAX_KNOWN_STATEMENTS).expect("Constant is nonzero"), @@ -323,39 +333,26 @@ where ); debug_assert!(_was_in.is_none()); }, - Event::NotificationStreamClosed { remote, protocol } - if protocol == self.protocol_name => - { - let _peer = self.peers.remove(&remote); + NotificationEvent::NotificationStreamClosed { peer } => { + let _peer = self.peers.remove(&peer); debug_assert!(_peer.is_some()); }, + NotificationEvent::NotificationReceived { peer, notification } => { + // Accept statements only when node is not major syncing + if self.sync.is_major_syncing() { + log::trace!( + target: LOG_TARGET, + "{peer}: Ignoring statements while major syncing or offline" + ); + return + } - Event::NotificationsReceived { remote, messages } => { - for (protocol, message) in messages { - if protocol != self.protocol_name { - continue - } - // Accept statements only when node is not major syncing - if self.sync.is_major_syncing() { - log::trace!( - target: LOG_TARGET, - "{remote}: Ignoring statements while major syncing or offline" - ); - continue - } - if let Ok(statements) = ::decode(&mut message.as_ref()) { - self.on_statements(remote, statements); - } else { - log::debug!( - target: LOG_TARGET, - "Failed to decode statement list from {remote}" - ); - } + if let Ok(statements) = ::decode(&mut notification.as_ref()) { + self.on_statements(peer, statements); + } else { + log::debug!(target: LOG_TARGET, "Failed to decode statement list from {peer}"); } }, - - // Not our concern. - Event::NotificationStreamOpened { .. } | Event::NotificationStreamClosed { .. } => {}, } } diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 2cb8eab22f7a..d7b024cd801c 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -38,7 +38,7 @@ use crate::{ warp::{EncodedProof, WarpProofRequest, WarpSyncParams}, }; -use codec::{Decode, Encode}; +use codec::{Decode, DecodeAll, Encode}; use futures::{ channel::oneshot, future::{BoxFuture, Fuse}, @@ -61,9 +61,12 @@ use sc_network::{ FullNetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode, NotificationHandshake, ProtocolId, SetConfig, }, + peer_store::{PeerStoreHandle, PeerStoreProvider}, request_responses::{IfDisconnected, RequestFailure}, + service::traits::{Direction, NotificationEvent, ValidationResult}, + types::ProtocolName, utils::LruHashSet, - NotificationsSink, ProtocolName, ReputationChange, + NotificationService, ReputationChange, }; use sc_network_common::{ role::Roles, @@ -88,15 +91,15 @@ use std::{ time::{Duration, Instant}, }; -/// Log target for this file. -const LOG_TARGET: &'static str = "sync"; - /// Interval at which we perform time based maintenance const TICK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(1100); /// Maximum number of known block hashes to keep for a peer. const MAX_KNOWN_BLOCKS: usize = 1024; // ~32kb per peer + LruHashSet overhead +/// Logging target for the file. +const LOG_TARGET: &str = "sync"; + /// If the block announces stream to peer has been inactive for 30 seconds meaning local node /// has not sent or received block announcements to/from the peer, report the node for inactivity, /// disconnect it and attempt to establish connection to some other peer. @@ -226,8 +229,6 @@ pub struct Peer { pub info: ExtendedPeerInfo, /// Holds a set of blocks known to this peer. pub known_blocks: LruHashSet, - /// Notification sink. - sink: NotificationsSink, /// Is the peer inbound. inbound: bool, } @@ -252,9 +253,6 @@ pub struct SyncingEngine { /// Channel for receiving service commands service_rx: TracingUnboundedReceiver>, - /// Channel for receiving inbound connections from `Protocol`. - sync_events_rx: sc_utils::mpsc::TracingUnboundedReceiver>, - /// Assigned roles. roles: Roles, @@ -312,12 +310,18 @@ pub struct SyncingEngine { /// Prometheus metrics. metrics: Option, + /// Handle that is used to communicate with `sc_network::Notifications`. + notification_service: Box, + /// When the syncing was started. /// /// Stored as an `Option` so once the initial wait has passed, `SyncingEngine` /// can reset the peer timers and continue with the normal eviction process. syncing_started: Option, + /// Handle to `PeerStore`. + peer_store_handle: PeerStoreHandle, + /// Instant when the last notification was sent or received. last_notification_io: Instant, @@ -362,7 +366,7 @@ where block_downloader: Arc>, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, - sync_events_rx: sc_utils::mpsc::TracingUnboundedReceiver>, + peer_store_handle: PeerStoreHandle, ) -> Result<(Self, SyncingService, NonDefaultSetConfig), ClientError> { let mode = net_config.network_config.sync_mode; let max_parallel_downloads = net_config.network_config.max_parallel_downloads; @@ -387,7 +391,7 @@ where } for config in net_config.notification_protocols() { let peer_ids = config - .set_config + .set_config() .reserved_nodes .iter() .map(|info| info.peer_id) @@ -438,7 +442,7 @@ where let warp_sync_target_block_header_rx_fused = warp_sync_target_block_header_rx .map_or(futures::future::pending().boxed().fuse(), |rx| rx.boxed().fuse()); - let block_announce_config = Self::get_block_announce_proto_config( + let (block_announce_config, notification_service) = Self::get_block_announce_proto_config( protocol_id, fork_id, roles, @@ -450,7 +454,6 @@ where .flatten() .expect("Genesis block exists; qed"), ); - let block_announce_protocol_name = block_announce_config.notifications_protocol.clone(); let chain_sync = ChainSync::new( mode, @@ -460,6 +463,7 @@ where warp_sync_config, )?; + let block_announce_protocol_name = block_announce_config.protocol_name().clone(); let (tx, service_rx) = tracing_unbounded("mpsc_chain_sync", 100_000); let num_connected = Arc::new(AtomicUsize::new(0)); let is_major_syncing = Arc::new(AtomicBool::new(false)); @@ -496,7 +500,6 @@ where num_connected: num_connected.clone(), is_major_syncing: is_major_syncing.clone(), service_rx, - sync_events_rx, genesis_hash, important_peers, default_peers_set_no_slot_connected_peers: HashSet::new(), @@ -508,8 +511,10 @@ where num_in_peers: 0usize, max_in_peers, event_streams: Vec::new(), + notification_service, tick_timeout, syncing_started: None, + peer_store_handle, last_notification_io: Instant::now(), metrics: if let Some(r) = metrics_registry { match Metrics::register(r, is_major_syncing.clone()) { @@ -673,23 +678,11 @@ where }; self.last_notification_io = Instant::now(); - peer.sink.send_sync_notification(message.encode()); + let _ = self.notification_service.send_sync_notification(peer_id, message.encode()); } } } - /// Inform sync about new best imported block. - pub fn new_best_block_imported(&mut self, hash: B::Hash, number: NumberFor) { - log::debug!(target: LOG_TARGET, "New best block imported {hash:?}/#{number}"); - - self.chain_sync.update_chain_info(&hash, number); - self.network_service.set_notification_handshake( - self.block_announce_protocol_name.clone(), - BlockAnnouncesHandshake::::build(self.roles, number, hash, self.genesis_hash) - .encode(), - ) - } - pub async fn run(mut self) { self.syncing_started = Some(Instant::now()); @@ -698,8 +691,10 @@ where _ = self.tick_timeout.tick() => self.perform_periodic_actions(), command = self.service_rx.select_next_some() => self.process_service_command(command), - sync_event = self.sync_events_rx.select_next_some() => - self.process_sync_event(sync_event), + notification_event = self.notification_service.next_event() => match notification_event { + Some(event) => self.process_notification_event(event), + None => return, + }, warp_target_block_header = &mut self.warp_sync_target_block_header_rx_fused => self.pass_warp_sync_target_block_header(warp_target_block_header), response_event = self.pending_responses.select_next_some() => @@ -853,8 +848,20 @@ where } }, ToServiceCommand::AnnounceBlock(hash, data) => self.announce_block(hash, data), - ToServiceCommand::NewBestBlockImported(hash, number) => - self.new_best_block_imported(hash, number), + ToServiceCommand::NewBestBlockImported(hash, number) => { + log::debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); + + self.chain_sync.update_chain_info(&hash, number); + let _ = self.notification_service.try_set_handshake( + BlockAnnouncesHandshake::::build( + self.roles, + number, + hash, + self.genesis_hash, + ) + .encode(), + ); + }, ToServiceCommand::Status(tx) => { let mut status = self.chain_sync.status(); status.num_connected_peers = self.peers.len() as u32; @@ -894,56 +901,60 @@ where } } - fn process_sync_event(&mut self, event: sc_network::SyncEvent) { + fn process_notification_event(&mut self, event: NotificationEvent) { match event { - sc_network::SyncEvent::NotificationStreamOpened { - remote, - received_handshake, - sink, - inbound, - tx, - } => match self.on_sync_peer_connected(remote, &received_handshake, sink, inbound) { - Ok(()) => { - let _ = tx.send(true); - }, - Err(()) => { - log::debug!( - target: LOG_TARGET, - "Failed to register peer {remote:?}: {received_handshake:?}", - ); - let _ = tx.send(false); - }, - }, - sc_network::SyncEvent::NotificationStreamClosed { remote } => { - if self.on_sync_peer_disconnected(remote).is_err() { - log::trace!( - target: LOG_TARGET, - "Disconnected peer which had earlier been refused by on_sync_peer_connected {}", - remote - ); - } + NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => { + let validation_result = self + .validate_connection(&peer, handshake, Direction::Inbound) + .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); + + let _ = result_tx.send(validation_result); }, - sc_network::SyncEvent::NotificationsReceived { remote, messages } => { - for message in messages { - if self.peers.contains_key(&remote) { - if let Ok(announce) = BlockAnnounce::decode(&mut message.as_ref()) { - self.last_notification_io = Instant::now(); - self.push_block_announce_validation(remote, announce); - } else { - log::warn!(target: "sub-libp2p", "Failed to decode block announce"); + NotificationEvent::NotificationStreamOpened { peer, handshake, direction, .. } => { + log::debug!( + target: LOG_TARGET, + "Substream opened for {peer}, handshake {handshake:?}" + ); + + match self.validate_connection(&peer, handshake, direction) { + Ok(handshake) => { + if self.on_sync_peer_connected(peer, &handshake, direction).is_err() { + log::debug!(target: LOG_TARGET, "Failed to register peer {peer}"); + self.network_service + .disconnect_peer(peer, self.block_announce_protocol_name.clone()); } - } else { - log::trace!( - target: LOG_TARGET, - "Received sync for peer earlier refused by sync layer: {remote}", - ); - } + }, + Err(wrong_genesis) => { + log::debug!(target: LOG_TARGET, "`SyncingEngine` rejected {peer}"); + + if wrong_genesis { + self.peer_store_handle.report_peer(peer, rep::GENESIS_MISMATCH); + } + + self.network_service + .disconnect_peer(peer, self.block_announce_protocol_name.clone()); + }, } }, - sc_network::SyncEvent::NotificationSinkReplaced { remote, sink } => { - if let Some(peer) = self.peers.get_mut(&remote) { - peer.sink = sink; + NotificationEvent::NotificationStreamClosed { peer } => { + self.on_sync_peer_disconnected(peer); + }, + NotificationEvent::NotificationReceived { peer, notification } => { + if !self.peers.contains_key(&peer) { + log::error!( + target: LOG_TARGET, + "received notification from {peer} who had been earlier refused by `SyncingEngine`", + ); + return } + + let Ok(announce) = BlockAnnounce::decode(&mut notification.as_ref()) else { + log::warn!(target: LOG_TARGET, "failed to decode block announce"); + return + }; + + self.last_notification_io = Instant::now(); + self.push_block_announce_validation(peer, announce); }, } } @@ -965,129 +976,167 @@ where /// Called by peer when it is disconnecting. /// /// Returns a result if the handshake of this peer was indeed accepted. - fn on_sync_peer_disconnected(&mut self, peer_id: PeerId) -> Result<(), ()> { - if let Some(info) = self.peers.remove(&peer_id) { - if self.important_peers.contains(&peer_id) { - log::warn!(target: LOG_TARGET, "Reserved peer {peer_id} disconnected"); - } else { - log::debug!(target: LOG_TARGET, "{peer_id} disconnected"); - } - - if !self.default_peers_set_no_slot_connected_peers.remove(&peer_id) && - info.inbound && info.info.roles.is_full() - { - match self.num_in_peers.checked_sub(1) { - Some(value) => { - self.num_in_peers = value; - }, - None => { - log::error!( - target: LOG_TARGET, - "trying to disconnect an inbound node which is not counted as inbound" - ); - debug_assert!(false); - }, - } - } - - self.chain_sync.peer_disconnected(&peer_id); + fn on_sync_peer_disconnected(&mut self, peer_id: PeerId) { + let Some(info) = self.peers.remove(&peer_id) else { + log::debug!(target: LOG_TARGET, "{peer_id} does not exist in `SyncingEngine`"); + return + }; - self.pending_responses.remove(&peer_id); - self.event_streams.retain(|stream| { - stream.unbounded_send(SyncEvent::PeerDisconnected(peer_id)).is_ok() - }); - Ok(()) + if self.important_peers.contains(&peer_id) { + log::warn!(target: LOG_TARGET, "Reserved peer {peer_id} disconnected"); } else { - Err(()) + log::debug!(target: LOG_TARGET, "{peer_id} disconnected"); } - } - /// Called on the first connection between two peers on the default set, after their exchange - /// of handshake. - /// - /// Returns `Ok` if the handshake is accepted and the peer added to the list of peers we sync - /// from. - fn on_sync_peer_connected( - &mut self, - peer_id: PeerId, - status: &BlockAnnouncesHandshake, - sink: NotificationsSink, - inbound: bool, - ) -> Result<(), ()> { - log::trace!(target: LOG_TARGET, "New peer {peer_id} {status:?}"); - - if self.peers.contains_key(&peer_id) { - log::error!( - target: LOG_TARGET, - "Called on_sync_peer_connected with already connected peer {peer_id}", - ); - debug_assert!(false); - return Err(()) + if !self.default_peers_set_no_slot_connected_peers.remove(&peer_id) && + info.inbound && info.info.roles.is_full() + { + match self.num_in_peers.checked_sub(1) { + Some(value) => { + self.num_in_peers = value; + }, + None => { + log::error!( + target: LOG_TARGET, + "trying to disconnect an inbound node which is not counted as inbound" + ); + debug_assert!(false); + }, + } } - if status.genesis_hash != self.genesis_hash { - self.network_service.report_peer(peer_id, rep::GENESIS_MISMATCH); + self.chain_sync.peer_disconnected(&peer_id); + self.pending_responses.remove(&peer_id); + self.event_streams + .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer_id)).is_ok()); + } + /// Validate received handshake. + fn validate_handshake( + &mut self, + peer_id: &PeerId, + handshake: Vec, + ) -> Result, bool> { + log::trace!(target: LOG_TARGET, "Validate handshake for {peer_id}"); + + let handshake = as DecodeAll>::decode_all(&mut &handshake[..]) + .map_err(|error| { + log::debug!(target: LOG_TARGET, "Failed to decode handshake for {peer_id}: {error:?}"); + false + })?; + + if handshake.genesis_hash != self.genesis_hash { if self.important_peers.contains(&peer_id) { log::error!( target: LOG_TARGET, - "Reserved peer id `{}` is on a different chain (our genesis: {} theirs: {})", - peer_id, + "Reserved peer id `{peer_id}` is on a different chain (our genesis: {} theirs: {})", self.genesis_hash, - status.genesis_hash, + handshake.genesis_hash, ); } else if self.boot_node_ids.contains(&peer_id) { log::error!( target: LOG_TARGET, - "Bootnode with peer id `{}` is on a different chain (our genesis: {} theirs: {})", - peer_id, + "Bootnode with peer id `{peer_id}` is on a different chain (our genesis: {} theirs: {})", self.genesis_hash, - status.genesis_hash, + handshake.genesis_hash, ); } else { log::debug!( target: LOG_TARGET, "Peer is on different chain (our genesis: {} theirs: {})", - self.genesis_hash, status.genesis_hash + self.genesis_hash, + handshake.genesis_hash ); } - return Err(()) + return Err(true) } - let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&peer_id); - let this_peer_reserved_slot: usize = if no_slot_peer { 1 } else { 0 }; + Ok(handshake) + } - // make sure to accept no more than `--in-peers` many full nodes - if !no_slot_peer && - status.roles.is_full() && - inbound && self.num_in_peers == self.max_in_peers - { - log::debug!( + /// Validate connection. + // NOTE Returning `Err(bool)` is a really ugly hack to work around the issue + // that `ProtocolController` thinks the peer is connected when in fact it can + // still be under validation. If the peer has different genesis than the + // local node the validation fails but the peer cannot be reported in + // `validate_connection()` as that is also called by + // `ValiateInboundSubstream` which means that the peer is still being + // validated and banning the peer when handling that event would + // result in peer getting dropped twice. + // + // The proper way to fix this is to integrate `ProtocolController` more + // tightly with `NotificationService` or add an additional API call for + // banning pre-accepted peers (which is not desirable) + fn validate_connection( + &mut self, + peer_id: &PeerId, + handshake: Vec, + direction: Direction, + ) -> Result, bool> { + log::trace!(target: LOG_TARGET, "New peer {peer_id} {handshake:?}"); + + let handshake = self.validate_handshake(peer_id, handshake)?; + + if self.peers.contains_key(&peer_id) { + log::error!( target: LOG_TARGET, - "All inbound slots have been consumed, rejecting {peer_id}", + "Called `validate_connection()` with already connected peer {peer_id}", ); - return Err(()) + debug_assert!(false); + return Err(false) } - if status.roles.is_full() && + let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&peer_id); + let this_peer_reserved_slot: usize = if no_slot_peer { 1 } else { 0 }; + + if handshake.roles.is_full() && self.chain_sync.num_peers() >= self.default_peers_set_num_full + self.default_peers_set_no_slot_connected_peers.len() + this_peer_reserved_slot { log::debug!(target: LOG_TARGET, "Too many full nodes, rejecting {peer_id}"); - return Err(()) + return Err(false) } - if status.roles.is_light() && + // make sure to accept no more than `--in-peers` many full nodes + if !no_slot_peer && + handshake.roles.is_full() && + direction.is_inbound() && + self.num_in_peers == self.max_in_peers + { + log::debug!(target: LOG_TARGET, "All inbound slots have been consumed, rejecting {peer_id}"); + return Err(false) + } + + // make sure that all slots are not occupied by light peers + // + // `ChainSync` only accepts full peers whereas `SyncingEngine` accepts both full and light + // peers. Verify that there is a slot in `SyncingEngine` for the inbound light peer + if handshake.roles.is_light() && (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light { - // Make sure that not all slots are occupied by light clients. log::debug!(target: LOG_TARGET, "Too many light nodes, rejecting {peer_id}"); - return Err(()) + return Err(false) } + Ok(handshake) + } + + /// Called on the first connection between two peers on the default set, after their exchange + /// of handshake. + /// + /// Returns `Ok` if the handshake is accepted and the peer added to the list of peers we sync + /// from. + fn on_sync_peer_connected( + &mut self, + peer_id: PeerId, + status: &BlockAnnouncesHandshake, + direction: Direction, + ) -> Result<(), ()> { + log::trace!(target: LOG_TARGET, "New peer {peer_id} {status:?}"); + let peer = Peer { info: ExtendedPeerInfo { roles: status.roles, @@ -1097,8 +1146,7 @@ where known_blocks: LruHashSet::new( NonZeroUsize::new(MAX_KNOWN_BLOCKS).expect("Constant is nonzero"), ), - sink, - inbound, + inbound: direction.is_inbound(), }; self.chain_sync.new_peer(peer_id, peer.info.best_hash, peer.info.best_number); @@ -1106,10 +1154,11 @@ where log::debug!(target: LOG_TARGET, "Connected {peer_id}"); self.peers.insert(peer_id, peer); + self.peer_store_handle.set_peer_role(&peer_id, status.roles.into()); - if no_slot_peer { + if self.default_peers_set_no_slot_peers.contains(&peer_id) { self.default_peers_set_no_slot_connected_peers.insert(peer_id); - } else if inbound && status.roles.is_full() { + } else if direction.is_inbound() && status.roles.is_full() { self.num_in_peers += 1; } @@ -1333,7 +1382,7 @@ where best_number: NumberFor, best_hash: B::Hash, genesis_hash: B::Hash, - ) -> NonDefaultSetConfig { + ) -> (NonDefaultSetConfig, Box) { let block_announces_protocol = { let genesis_hash = genesis_hash.as_ref(); if let Some(ref fork_id) = fork_id { @@ -1347,14 +1396,11 @@ where } }; - NonDefaultSetConfig { - notifications_protocol: block_announces_protocol.into(), - fallback_names: iter::once( - format!("/{}/block-announces/1", protocol_id.as_ref()).into(), - ) - .collect(), - max_notification_size: MAX_BLOCK_ANNOUNCE_SIZE, - handshake: Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( + NonDefaultSetConfig::new( + block_announces_protocol.into(), + iter::once(format!("/{}/block-announces/1", protocol_id.as_ref()).into()).collect(), + MAX_BLOCK_ANNOUNCE_SIZE, + Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( roles, best_number, best_hash, @@ -1362,13 +1408,13 @@ where ))), // NOTE: `set_config` will be ignored by `protocol.rs` as the block announcement // protocol is still hardcoded into the peerset. - set_config: SetConfig { + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ) } /// Import blocks. diff --git a/substrate/client/network/sync/src/service/mock.rs b/substrate/client/network/sync/src/service/mock.rs index 885eb1f8da59..47986a71d012 100644 --- a/substrate/client/network/sync/src/service/mock.rs +++ b/substrate/client/network/sync/src/service/mock.rs @@ -27,6 +27,7 @@ use sc_network::{ NetworkNotification, NetworkPeers, NetworkRequest, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT, ReputationChange, }; +use sc_network_common::role::ObservedRole; use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::collections::HashSet; @@ -105,6 +106,7 @@ mockall::mock! { peers: Vec ) -> Result<(), String>; fn sync_num_connected(&self) -> usize; + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option; } #[async_trait::async_trait] diff --git a/substrate/client/network/test/src/lib.rs b/substrate/client/network/test/src/lib.rs index cfc3cb7af3fc..71f13b74a532 100644 --- a/substrate/client/network/test/src/lib.rs +++ b/substrate/client/network/test/src/lib.rs @@ -58,7 +58,7 @@ use sc_network::{ request_responses::ProtocolConfig as RequestResponseConfig, types::ProtocolName, Multiaddr, NetworkBlock, NetworkService, NetworkStateInfo, NetworkSyncForkRequest, - NetworkWorker, + NetworkWorker, NotificationService, }; use sc_network_common::role::Roles; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; @@ -239,6 +239,7 @@ pub struct Peer { imported_blocks_stream: Pin> + Send>>, finality_notification_stream: Pin> + Send>>, listen_addr: Multiaddr, + notification_services: HashMap>, } impl Peer @@ -263,8 +264,8 @@ where } /// Returns the number of peers we're connected to. - pub fn num_peers(&self) -> usize { - self.network.num_connected_peers() + pub async fn num_peers(&self) -> usize { + self.sync_service.status().await.unwrap().num_connected_peers as usize } /// Returns the number of downloaded blocks. @@ -502,10 +503,19 @@ where self.network.service() } + /// Get `SyncingService`. pub fn sync_service(&self) -> &Arc> { &self.sync_service } + /// Take notification handle for enabled protocol. + pub fn take_notification_service( + &mut self, + protocol: &ProtocolName, + ) -> Option> { + self.notification_services.remove(protocol) + } + /// Get a reference to the network worker. pub fn network(&self) -> &NetworkWorker::Hash> { &self.network @@ -778,6 +788,23 @@ pub trait TestNetFactory: Default + Sized + Send { network_config.transport = TransportConfig::MemoryOnly; network_config.listen_addresses = vec![listen_addr.clone()]; network_config.allow_non_globals_in_dht = true; + + let (notif_configs, notif_handles): (Vec<_>, Vec<_>) = config + .notifications_protocols + .into_iter() + .map(|p| { + let (config, handle) = NonDefaultSetConfig::new( + p.clone(), + Vec::new(), + 1024 * 1024, + None, + Default::default(), + ); + + (config, (p, handle)) + }) + .unzip(); + if let Some(connect_to) = config.connect_to_peers { let addrs = connect_to .iter() @@ -849,11 +876,16 @@ pub trait TestNetFactory: Default + Sized + Send { protocol_config }; + let peer_store = PeerStore::new( + network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), + ); + let peer_store_handle = peer_store.handle(); + self.spawn_task(peer_store.run().boxed()); + let block_announce_validator = config .block_announce_validator .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)); - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, sync_service, block_announce_config) = sc_network_sync::engine::SyncingEngine::new( Roles::from(if config.is_authority { &Role::Authority } else { &Role::Full }), @@ -869,7 +901,7 @@ pub trait TestNetFactory: Default + Sized + Send { block_relay_params.downloader, state_request_protocol_config.name.clone(), Some(warp_protocol_config.name.clone()), - rx, + peer_store_handle.clone(), ) .unwrap(); let sync_service_import_queue = Box::new(sync_service.clone()); @@ -887,22 +919,10 @@ pub trait TestNetFactory: Default + Sized + Send { full_net_config.add_request_response_protocol(config); } - for protocol in config.notifications_protocols { - full_net_config.add_notification_protocol(NonDefaultSetConfig { - notifications_protocol: protocol, - fallback_names: Vec::new(), - max_notification_size: 1024 * 1024, - handshake: None, - set_config: Default::default(), - }); + for config in notif_configs { + full_net_config.add_notification_protocol(config); } - let peer_store = PeerStore::new( - network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), - ); - let peer_store_handle = peer_store.handle(); - self.spawn_task(peer_store.run().boxed()); - let genesis_hash = client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed"); let network = NetworkWorker::new(sc_network::config::Params { @@ -917,7 +937,6 @@ pub trait TestNetFactory: Default + Sized + Send { fork_id, metrics_registry: None, block_announce_config, - tx, }) .unwrap(); @@ -953,6 +972,7 @@ pub trait TestNetFactory: Default + Sized + Send { backend: Some(backend), imported_blocks_stream, finality_notification_stream, + notification_services: HashMap::from_iter(notif_handles.into_iter()), block_import, verifier, network, @@ -967,20 +987,6 @@ pub trait TestNetFactory: Default + Sized + Send { tokio::spawn(f); } - /// Polls the testnet until all peers are connected to each other. - /// - /// Must be executed in a task context. - fn poll_until_connected(&mut self, cx: &mut FutureContext) -> Poll<()> { - self.poll(cx); - - let num_peers = self.peers().len(); - if self.peers().iter().all(|p| p.num_peers() == num_peers - 1) { - return Poll::Ready(()) - } - - Poll::Pending - } - async fn is_in_sync(&mut self) -> bool { let mut highest = None; let peers = self.peers_mut(); @@ -1058,10 +1064,27 @@ pub trait TestNetFactory: Default + Sized + Send { } /// Run the network until all peers are connected to each other. - /// - /// Calls `poll_until_connected` repeatedly with the runtime passed as parameter. async fn run_until_connected(&mut self) { - futures::future::poll_fn::<(), _>(|cx| self.poll_until_connected(cx)).await; + let num_peers = self.peers().len(); + let sync_services = + self.peers().iter().map(|info| info.sync_service.clone()).collect::>(); + + 'outer: loop { + for sync_service in &sync_services { + if sync_service.status().await.unwrap().num_connected_peers as usize != + num_peers - 1 + { + futures::future::poll_fn::<(), _>(|cx| { + self.poll(cx); + Poll::Ready(()) + }) + .await; + continue 'outer + } + } + + break + } } /// Polls the testnet. Processes all the pending actions. diff --git a/substrate/client/network/test/src/service.rs b/substrate/client/network/test/src/service.rs index 62d7f9f9d1bb..800c0d4369c2 100644 --- a/substrate/client/network/test/src/service.rs +++ b/substrate/client/network/test/src/service.rs @@ -24,8 +24,9 @@ use sc_network::{ config::{self, FullNetworkConfiguration, MultiaddrWithPeerId, ProtocolId, TransportConfig}, event::Event, peer_store::PeerStore, - NetworkEventStream, NetworkNotification, NetworkPeers, NetworkService, NetworkStateInfo, - NetworkWorker, + service::traits::{NotificationEvent, ValidationResult}, + NetworkEventStream, NetworkPeers, NetworkService, NetworkStateInfo, NetworkWorker, + NotificationService, }; use sc_network_common::role::Roles; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; @@ -116,7 +117,7 @@ impl TestNetworkBuilder { self } - pub fn build(mut self) -> TestNetwork { + pub fn build(mut self) -> (TestNetwork, Option>) { let client = self.client.as_mut().map_or( Arc::new(TestClientBuilder::with_default_backend().build_with_longest_chain().0), |v| v.clone(), @@ -183,7 +184,12 @@ impl TestNetworkBuilder { protocol_config }; - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); + let peer_store = PeerStore::new( + network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), + ); + let peer_store_handle = peer_store.handle(); + tokio::spawn(peer_store.run().boxed()); + let (engine, chain_sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config::Role::Full), client.clone(), @@ -198,24 +204,27 @@ impl TestNetworkBuilder { block_relay_params.downloader, state_request_protocol_config.name.clone(), None, - rx, + peer_store_handle.clone(), ) .unwrap(); let mut link = self.link.unwrap_or(Box::new(chain_sync_service.clone())); - if !self.notification_protocols.is_empty() { + let handle = if !self.notification_protocols.is_empty() { for config in self.notification_protocols { full_net_config.add_notification_protocol(config); } + None } else { - full_net_config.add_notification_protocol(config::NonDefaultSetConfig { - notifications_protocol: PROTOCOL_NAME.into(), - fallback_names: Vec::new(), - max_notification_size: 1024 * 1024, - handshake: None, - set_config: self.set_config.unwrap_or_default(), - }); - } + let (config, handle) = config::NonDefaultSetConfig::new( + PROTOCOL_NAME.into(), + Vec::new(), + 1024 * 1024, + None, + self.set_config.unwrap_or_default(), + ); + full_net_config.add_notification_protocol(config); + Some(handle) + }; for config in [ block_relay_params.request_response_config, @@ -225,12 +234,6 @@ impl TestNetworkBuilder { full_net_config.add_request_response_protocol(config); } - let peer_store = PeerStore::new( - network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), - ); - let peer_store_handle = peer_store.handle(); - tokio::spawn(peer_store.run().boxed()); - let genesis_hash = client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed"); let worker = NetworkWorker::< @@ -248,7 +251,6 @@ impl TestNetworkBuilder { protocol_id, fork_id, metrics_registry: None, - tx, }) .unwrap(); @@ -268,7 +270,7 @@ impl TestNetworkBuilder { }); tokio::spawn(engine.run()); - TestNetwork::new(worker) + (TestNetwork::new(worker), handle) } } @@ -276,18 +278,18 @@ impl TestNetworkBuilder { /// The nodes are connected together and have the `PROTOCOL_NAME` protocol registered. fn build_nodes_one_proto() -> ( Arc, - impl Stream, + Option>, Arc, - impl Stream, + Option>, ) { let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (node1, events_stream1) = TestNetworkBuilder::new() + let (network1, handle1) = TestNetworkBuilder::new() .with_listen_addresses(vec![listen_addr.clone()]) - .build() - .start_network(); + .build(); + let (node1, _) = network1.start_network(); - let (node2, events_stream2) = TestNetworkBuilder::new() + let (network2, handle2) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr, @@ -295,10 +297,11 @@ fn build_nodes_one_proto() -> ( }], ..Default::default() }) - .build() - .start_network(); + .build(); + + let (node2, _) = network2.start_network(); - (node1, events_stream1, node2, events_stream2) + (node1, handle1, node2, handle2) } #[tokio::test] @@ -306,22 +309,15 @@ async fn notifications_state_consistent() { // Runs two nodes and ensures that events are propagated out of the API in a consistent // correct order, which means no notification received on a closed substream. - let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); + let (node1, handle1, node2, handle2) = build_nodes_one_proto(); + let (mut handle1, mut handle2) = (handle1.unwrap(), handle2.unwrap()); // Write some initial notifications that shouldn't get through. for _ in 0..(rand::random::() % 5) { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1.send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } for _ in 0..(rand::random::() % 5) { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2.send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } // True if we have an active substream from node1 to node2. @@ -343,18 +339,10 @@ async fn notifications_state_consistent() { // Start by sending a notification from node1 to node2 and vice-versa. Part of the // test consists in ensuring that notifications get ignored if the stream isn't open. if rand::random::() % 5 >= 3 { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1.send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } if rand::random::() % 5 >= 3 { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2.send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } // Also randomly disconnect the two nodes from time to time. @@ -367,8 +355,8 @@ async fn notifications_state_consistent() { // Grab next event from either `events_stream1` or `events_stream2`. let next_event = { - let next1 = events_stream1.next(); - let next2 = events_stream2.next(); + let next1 = handle1.next_event(); + let next2 = handle2.next_event(); // We also await on a small timer, otherwise it is possible for the test to wait // forever while nothing at all happens on the network. let continue_test = futures_timer::Delay::new(Duration::from_millis(20)); @@ -383,58 +371,55 @@ async fn notifications_state_consistent() { }; match next_event { - future::Either::Left(Event::NotificationStreamOpened { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - something_happened = true; - assert!(!node1_to_node2_open); - node1_to_node2_open = true; - assert_eq!(remote, node2.local_peer_id()); - }, - future::Either::Right(Event::NotificationStreamOpened { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - something_happened = true; - assert!(!node2_to_node1_open); - node2_to_node1_open = true; - assert_eq!(remote, node1.local_peer_id()); - }, - future::Either::Left(Event::NotificationStreamClosed { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - assert!(node1_to_node2_open); - node1_to_node2_open = false; - assert_eq!(remote, node2.local_peer_id()); - }, - future::Either::Right(Event::NotificationStreamClosed { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - assert!(node2_to_node1_open); - node2_to_node1_open = false; - assert_eq!(remote, node1.local_peer_id()); - }, - future::Either::Left(Event::NotificationsReceived { remote, .. }) => { + future::Either::Left(NotificationEvent::ValidateInboundSubstream { + result_tx, .. + }) => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + future::Either::Right(NotificationEvent::ValidateInboundSubstream { + result_tx, + .. + }) => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + future::Either::Left(NotificationEvent::NotificationStreamOpened { peer, .. }) => { + something_happened = true; + assert!(!node1_to_node2_open); + node1_to_node2_open = true; + assert_eq!(peer, node2.local_peer_id()); + }, + future::Either::Right(NotificationEvent::NotificationStreamOpened { peer, .. }) => { + something_happened = true; + assert!(!node2_to_node1_open); + node2_to_node1_open = true; + assert_eq!(peer, node1.local_peer_id()); + }, + future::Either::Left(NotificationEvent::NotificationStreamClosed { peer, .. }) => { assert!(node1_to_node2_open); - assert_eq!(remote, node2.local_peer_id()); + node1_to_node2_open = false; + assert_eq!(peer, node2.local_peer_id()); + }, + future::Either::Right(NotificationEvent::NotificationStreamClosed { peer, .. }) => { + assert!(node2_to_node1_open); + node2_to_node1_open = false; + assert_eq!(peer, node1.local_peer_id()); + }, + future::Either::Left(NotificationEvent::NotificationReceived { peer, .. }) => { + assert!(node1_to_node2_open); + assert_eq!(peer, node2.local_peer_id()); if rand::random::() % 5 >= 4 { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1 + .send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } }, - future::Either::Right(Event::NotificationsReceived { remote, .. }) => { + future::Either::Right(NotificationEvent::NotificationReceived { peer, .. }) => { assert!(node2_to_node1_open); - assert_eq!(remote, node1.local_peer_id()); + assert_eq!(peer, node1.local_peer_id()); if rand::random::() % 5 >= 4 { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2 + .send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } }, - - // Add new events here. - future::Either::Left(Event::Dht(_)) => {}, - future::Either::Right(Event::Dht(_)) => {}, }; } } @@ -444,20 +429,29 @@ async fn lots_of_incoming_peers_works() { sp_tracing::try_init_simple(); let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (main_node, _) = TestNetworkBuilder::new() + let (main_node, handle1) = TestNetworkBuilder::new() .with_listen_addresses(vec![listen_addr.clone()]) .with_set_config(config::SetConfig { in_peers: u32::MAX, ..Default::default() }) - .build() - .start_network(); + .build(); + let mut handle1 = handle1.unwrap(); + let (main_node, _) = main_node.start_network(); let main_node_peer_id = main_node.local_peer_id(); + tokio::spawn(async move { + while let Some(event) = handle1.next_event().await { + if let NotificationEvent::ValidateInboundSubstream { result_tx, .. } = event { + result_tx.send(ValidationResult::Accept).unwrap(); + } + } + }); + // We spawn background tasks and push them in this `Vec`. They will all be waited upon before // this test ends. let mut background_tasks_to_wait = Vec::new(); for _ in 0..32 { - let (_dialing_node, event_stream) = TestNetworkBuilder::new() + let (dialing_node, handle) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr.clone(), @@ -465,8 +459,9 @@ async fn lots_of_incoming_peers_works() { }], ..Default::default() }) - .build() - .start_network(); + .build(); + let mut handle = handle.unwrap(); + let (_, _) = dialing_node.start_network(); background_tasks_to_wait.push(tokio::spawn(async move { // Create a dummy timer that will "never" fire, and that will be overwritten when we @@ -474,34 +469,23 @@ async fn lots_of_incoming_peers_works() { // make the code below way more complicated. let mut timer = futures_timer::Delay::new(Duration::from_secs(3600 * 24 * 7)).fuse(); - let mut event_stream = event_stream.fuse(); - let mut sync_protocol_name = None; loop { futures::select! { _ = timer => { // Test succeeds when timer fires. return; } - ev = event_stream.next() => { - match ev.unwrap() { - Event::NotificationStreamOpened { protocol, remote, .. } => { - if let None = sync_protocol_name { - sync_protocol_name = Some(protocol.clone()); - } - - assert_eq!(remote, main_node_peer_id); - // Test succeeds after 5 seconds. This timer is here in order to - // detect a potential problem after opening. - timer = futures_timer::Delay::new(Duration::from_secs(5)).fuse(); - } - Event::NotificationStreamClosed { protocol, .. } => { - if Some(protocol) != sync_protocol_name { - // Test failed. - panic!(); - } - } - _ => {} + ev = handle.next_event().fuse() => match ev.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); } + NotificationEvent::NotificationStreamOpened { peer, .. } => { + assert_eq!(peer, main_node_peer_id); + // Test succeeds after 5 seconds. This timer is here in order to + // detect a potential problem after opening. + timer = futures_timer::Delay::new(Duration::from_secs(5)).fuse(); + } + _ => {} } } } @@ -518,33 +502,27 @@ async fn notifications_back_pressure() { const TOTAL_NOTIFS: usize = 10_000; - let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); + let (_node1, handle1, node2, handle2) = build_nodes_one_proto(); + let (mut handle1, mut handle2) = (handle1.unwrap(), handle2.unwrap()); let node2_id = node2.local_peer_id(); let receiver = tokio::spawn(async move { let mut received_notifications = 0; - let mut sync_protocol_name = None; while received_notifications < TOTAL_NOTIFS { - match events_stream2.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, .. } => { - if let None = sync_protocol_name { - sync_protocol_name = Some(protocol); - } + match handle2.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); }, - Event::NotificationStreamClosed { protocol, .. } => { - if Some(&protocol) != sync_protocol_name.as_ref() { - panic!() - } + NotificationEvent::NotificationReceived { notification, .. } => { + assert_eq!( + notification, + format!("hello #{}", received_notifications).into_bytes() + ); + received_notifications += 1; }, - Event::NotificationsReceived { messages, .. } => - for message in messages { - assert_eq!(message.0, PROTOCOL_NAME.into()); - assert_eq!(message.1, format!("hello #{}", received_notifications)); - received_notifications += 1; - }, _ => {}, - }; + } if rand::random::() < 2 { tokio::time::sleep(Duration::from_millis(rand::random::() % 750)).await; @@ -554,20 +532,20 @@ async fn notifications_back_pressure() { // Wait for the `NotificationStreamOpened`. loop { - match events_stream1.next().await.unwrap() { - Event::NotificationStreamOpened { .. } => break, + match handle1.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { .. } => break, _ => {}, }; } // Sending! for num in 0..TOTAL_NOTIFS { - let notif = node1.notification_sender(node2_id, PROTOCOL_NAME.into()).unwrap(); - notif - .ready() + handle1 + .send_async_notification(&node2_id, format!("hello #{}", num).into_bytes()) .await - .unwrap() - .send(format!("hello #{}", num).into_bytes()) .unwrap(); } @@ -576,28 +554,31 @@ async fn notifications_back_pressure() { #[tokio::test] async fn fallback_name_working() { + sp_tracing::try_init_simple(); // Node 1 supports the protocols "new" and "old". Node 2 only supports "old". Checks whether // they can connect. const NEW_PROTOCOL_NAME: &str = "/new-shiny-protocol-that-isnt-PROTOCOL_NAME"; let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (node1, mut events_stream1) = TestNetworkBuilder::new() - .with_notification_protocol(config::NonDefaultSetConfig { - notifications_protocol: NEW_PROTOCOL_NAME.into(), - fallback_names: vec![PROTOCOL_NAME.into()], - max_notification_size: 1024 * 1024, - handshake: None, - set_config: Default::default(), - }) + let (config, mut handle1) = config::NonDefaultSetConfig::new( + NEW_PROTOCOL_NAME.into(), + vec![PROTOCOL_NAME.into()], + 1024 * 1024, + None, + Default::default(), + ); + let (network1, _) = TestNetworkBuilder::new() + .with_notification_protocol(config) .with_config(config::NetworkConfiguration { listen_addresses: vec![listen_addr.clone()], transport: TransportConfig::MemoryOnly, ..config::NetworkConfiguration::new_local() }) - .build() - .start_network(); + .build(); - let (_, mut events_stream2) = TestNetworkBuilder::new() + let (node1, _) = network1.start_network(); + + let (network2, handle2) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr, @@ -605,34 +586,38 @@ async fn fallback_name_working() { }], ..Default::default() }) - .build() - .start_network(); + .build(); + let mut handle2 = handle2.unwrap(); + let _ = network2.start_network(); let receiver = tokio::spawn(async move { // Wait for the `NotificationStreamOpened`. loop { - match events_stream2.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, negotiated_fallback, .. } => { - assert_eq!(protocol, PROTOCOL_NAME.into()); + match handle2.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { negotiated_fallback, .. } => { assert_eq!(negotiated_fallback, None); break }, _ => {}, - }; + } } }); // Wait for the `NotificationStreamOpened`. loop { - match events_stream1.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, negotiated_fallback, .. } - if protocol == NEW_PROTOCOL_NAME.into() => - { + match handle1.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { negotiated_fallback, .. } => { assert_eq!(negotiated_fallback, Some(PROTOCOL_NAME.into())); break }, _ => {}, - }; + } } receiver.await.unwrap(); @@ -655,6 +640,7 @@ async fn ensure_listen_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -674,6 +660,7 @@ async fn ensure_listen_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -699,6 +686,7 @@ async fn ensure_boot_node_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -723,6 +711,7 @@ async fn ensure_boot_node_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -751,6 +740,7 @@ async fn ensure_reserved_node_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -778,6 +768,7 @@ async fn ensure_reserved_node_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -800,6 +791,7 @@ async fn ensure_public_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -821,5 +813,6 @@ async fn ensure_public_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } diff --git a/substrate/client/network/test/src/sync.rs b/substrate/client/network/test/src/sync.rs index 389177b4aaf1..f2be662ada16 100644 --- a/substrate/client/network/test/src/sync.rs +++ b/substrate/client/network/test/src/sync.rs @@ -44,16 +44,16 @@ async fn sync_peers_works() { sp_tracing::try_init_simple(); let mut net = TestNet::new(3); - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - for peer in 0..3 { - if net.peer(peer).num_peers() != 2 { - return Poll::Pending - } - } - Poll::Ready(()) - }) - .await; + while net.peer(0).num_peers().await != 2 && + net.peer(1).num_peers().await != 2 && + net.peer(2).num_peers().await != 2 + { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); + Poll::Ready(()) + }) + .await; + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -412,15 +412,13 @@ async fn can_sync_small_non_best_forks() { assert!(net.peer(1).client().header(small_hash).unwrap().is_none()); // poll until the two nodes connect, otherwise announcing the block will not work - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } // synchronization: 0 synced to longer chain and 1 didn't sync to small chain. @@ -465,6 +463,7 @@ async fn can_sync_forks_ahead_of_the_best_chain() { net.peer(1).push_blocks(1, false); net.run_until_connected().await; + // Peer 0 is on 2-block fork which is announced with is_best=false let fork_hash = net .peer(0) @@ -516,15 +515,13 @@ async fn can_sync_explicit_forks() { assert!(net.peer(1).client().header(small_hash).unwrap().is_none()); // poll until the two nodes connect, otherwise announcing the block will not work - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } // synchronization: 0 synced to longer chain and 1 didn't sync to small chain. @@ -613,15 +610,14 @@ async fn full_sync_requires_block_body() { net.peer(0).push_headers(1); // Wait for nodes to connect - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } + net.run_until_idle().await; assert_eq!(net.peer(1).client.info().best_number, 0); } @@ -917,18 +913,16 @@ async fn block_announce_data_is_propagated() { }); // Wait until peer 1 is connected to both nodes. - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(1).num_peers() == 2 && - net.peer(0).num_peers() == 1 && - net.peer(2).num_peers() == 1 - { + while net.peer(1).num_peers().await != 2 || + net.peer(0).num_peers().await != 1 || + net.peer(2).num_peers().await != 1 + { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } else { - Poll::Pending - } - }) - .await; + }) + .await; + } let block_hash = net .peer(0) @@ -1010,7 +1004,7 @@ async fn multiple_requests_are_accepted_as_long_as_they_are_not_fulfilled() { tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; net.peer(0).push_blocks(1, false); net.run_until_sync().await; - assert_eq!(1, net.peer(0).num_peers()); + assert_eq!(1, net.peer(0).num_peers().await); } let hashof10 = hashes[9]; diff --git a/substrate/client/network/transactions/src/lib.rs b/substrate/client/network/transactions/src/lib.rs index 1b97d4b96c97..9758ea4c4fcb 100644 --- a/substrate/client/network/transactions/src/lib.rs +++ b/substrate/client/network/transactions/src/lib.rs @@ -21,8 +21,8 @@ //! Usage: //! //! - Use [`TransactionsHandlerPrototype::new`] to create a prototype. -//! - Pass the return value of [`TransactionsHandlerPrototype::set_config`] to the network -//! configuration as an extra peers set. +//! - Pass the `NonDefaultSetConfig` returned from [`TransactionsHandlerPrototype::new`] to the +//! network configuration as an extra peers set. //! - Use [`TransactionsHandlerPrototype::build`] then [`TransactionsHandler::run`] to obtain a //! `Future` that processes transactions. @@ -37,7 +37,7 @@ use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; use sc_network::{ config::{NonDefaultSetConfig, NonReservedPeerMode, ProtocolId, SetConfig}, error, - event::Event, + service::traits::{NotificationEvent, NotificationService, ValidationResult}, types::ProtocolName, utils::{interval, LruHashSet}, NetworkEventStream, NetworkNotification, NetworkPeers, @@ -115,8 +115,11 @@ impl Future for PendingTransaction { /// Prototype for a [`TransactionsHandler`]. pub struct TransactionsHandlerPrototype { + /// Name of the transaction protocol. protocol_name: ProtocolName, - fallback_protocol_names: Vec, + + /// Handle that is used to communicate with `sc_network::Notifications`. + notification_service: Box, } impl TransactionsHandlerPrototype { @@ -125,35 +128,28 @@ impl TransactionsHandlerPrototype { protocol_id: ProtocolId, genesis_hash: Hash, fork_id: Option<&str>, - ) -> Self { + ) -> (Self, NonDefaultSetConfig) { let genesis_hash = genesis_hash.as_ref(); - let protocol_name = if let Some(fork_id) = fork_id { + let protocol_name: ProtocolName = if let Some(fork_id) = fork_id { format!("/{}/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash), fork_id) } else { format!("/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash)) - }; - let legacy_protocol_name = format!("/{}/transactions/1", protocol_id.as_ref()); - - Self { - protocol_name: protocol_name.into(), - fallback_protocol_names: iter::once(legacy_protocol_name.into()).collect(), } - } - - /// Returns the configuration of the set to put in the network configuration. - pub fn set_config(&self) -> NonDefaultSetConfig { - NonDefaultSetConfig { - notifications_protocol: self.protocol_name.clone(), - fallback_names: self.fallback_protocol_names.clone(), - max_notification_size: MAX_TRANSACTIONS_SIZE, - handshake: None, - set_config: SetConfig { + .into(); + let (config, notification_service) = NonDefaultSetConfig::new( + protocol_name.clone(), + vec![format!("/{}/transactions/1", protocol_id.as_ref()).into()], + MAX_TRANSACTIONS_SIZE, + None, + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ); + + (Self { protocol_name, notification_service }, config) } /// Turns the prototype into the actual handler. Returns a controller that allows controlling @@ -173,12 +169,12 @@ impl TransactionsHandlerPrototype { transaction_pool: Arc>, metrics_registry: Option<&Registry>, ) -> error::Result<(TransactionsHandler, TransactionsHandlerController)> { - let net_event_stream = network.event_stream("transactions-handler-net"); let sync_event_stream = sync.event_stream("transactions-handler-sync"); let (to_handler, from_controller) = tracing_unbounded("mpsc_transactions_handler", 100_000); let handler = TransactionsHandler { protocol_name: self.protocol_name, + notification_service: self.notification_service, propagate_timeout: (Box::pin(interval(PROPAGATE_TIMEOUT)) as Pin + Send>>) .fuse(), @@ -186,7 +182,6 @@ impl TransactionsHandlerPrototype { pending_transactions_peers: HashMap::new(), network, sync, - net_event_stream: net_event_stream.fuse(), sync_event_stream: sync_event_stream.fuse(), peers: HashMap::new(), transaction_pool, @@ -253,8 +248,6 @@ pub struct TransactionsHandler< network: N, /// Syncing service. sync: S, - /// Stream of networking events. - net_event_stream: stream::Fuse + Send>>>, /// Receiver for syncing-related events. sync_event_stream: stream::Fuse + Send>>>, // All connected peers @@ -263,6 +256,8 @@ pub struct TransactionsHandler< from_controller: TracingUnboundedReceiver>, /// Prometheus metrics. metrics: Option, + /// Handle that is used to communicate with `sc_network::Notifications`. + notification_service: Box, } /// Peer information @@ -295,14 +290,6 @@ where warn!(target: "sub-libp2p", "Inconsistent state, no peers for pending transaction!"); } }, - network_event = self.net_event_stream.next() => { - if let Some(network_event) = network_event { - self.handle_network_event(network_event).await; - } else { - // Networking has seemingly closed. Closing as well. - return; - } - }, sync_event = self.sync_event_stream.next() => { if let Some(sync_event) = sync_event { self.handle_sync_event(sync_event); @@ -317,10 +304,61 @@ where ToHandler::PropagateTransactions => self.propagate_transactions(), } }, + event = self.notification_service.next_event().fuse() => { + if let Some(event) = event { + self.handle_notification_event(event) + } else { + // `Notifications` has seemingly closed. Closing as well. + return + } + } } } } + fn handle_notification_event(&mut self, event: NotificationEvent) { + match event { + NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx, .. } => { + // only accept peers whose role can be determined + let result = self + .network + .peer_role(peer, handshake) + .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); + let _ = result_tx.send(result); + }, + NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { + let Some(role) = self.network.peer_role(peer, handshake) else { + log::debug!(target: "sub-libp2p", "role for {peer} couldn't be determined"); + return + }; + + let _was_in = self.peers.insert( + peer, + Peer { + known_transactions: LruHashSet::new( + NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS).expect("Constant is nonzero"), + ), + role, + }, + ); + debug_assert!(_was_in.is_none()); + }, + NotificationEvent::NotificationStreamClosed { peer } => { + let _peer = self.peers.remove(&peer); + debug_assert!(_peer.is_some()); + }, + NotificationEvent::NotificationReceived { peer, notification } => { + if let Ok(m) = + as Decode>::decode(&mut notification.as_ref()) + { + self.on_transactions(peer, m); + } else { + warn!(target: "sub-libp2p", "Failed to decode transactions list"); + } + }, + } + } + fn handle_sync_event(&mut self, event: SyncEvent) { match event { SyncEvent::PeerConnected(remote) => { @@ -346,51 +384,6 @@ where } } - async fn handle_network_event(&mut self, event: Event) { - match event { - Event::Dht(_) => {}, - Event::NotificationStreamOpened { remote, protocol, role, .. } - if protocol == self.protocol_name => - { - let _was_in = self.peers.insert( - remote, - Peer { - known_transactions: LruHashSet::new( - NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS).expect("Constant is nonzero"), - ), - role, - }, - ); - debug_assert!(_was_in.is_none()); - }, - Event::NotificationStreamClosed { remote, protocol } - if protocol == self.protocol_name => - { - let _peer = self.peers.remove(&remote); - debug_assert!(_peer.is_some()); - }, - - Event::NotificationsReceived { remote, messages } => { - for (protocol, message) in messages { - if protocol != self.protocol_name { - continue - } - - if let Ok(m) = - as Decode>::decode(&mut message.as_ref()) - { - self.on_transactions(remote, m); - } else { - warn!(target: "sub-libp2p", "Failed to decode transactions list"); - } - } - }, - - // Not our concern. - Event::NotificationStreamOpened { .. } | Event::NotificationStreamClosed { .. } => {}, - } - } - /// Called when peer sends us new transactions fn on_transactions(&mut self, who: PeerId, transactions: Transactions) { // Accept transactions only when node is not major syncing @@ -482,8 +475,7 @@ where propagated_to.entry(hash).or_default().push(who.to_base58()); } trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who); - self.network - .write_notification(*who, self.protocol_name.clone(), to_send.encode()); + let _ = self.notification_service.send_sync_notification(who, to_send.encode()); } } diff --git a/substrate/client/offchain/src/api.rs b/substrate/client/offchain/src/api.rs index c7df5784d329..2901bab2f267 100644 --- a/substrate/client/offchain/src/api.rs +++ b/substrate/client/offchain/src/api.rs @@ -223,7 +223,7 @@ mod tests { use sc_client_db::offchain::LocalStorage; use sc_network::{ config::MultiaddrWithPeerId, types::ProtocolName, NetworkPeers, NetworkStateInfo, - ReputationChange, + ObservedRole, ReputationChange, }; use sp_core::offchain::{storage::OffchainDb, DbExternalities, Externalities, StorageKind}; use std::time::SystemTime; @@ -294,6 +294,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } impl NetworkStateInfo for TestNetwork { diff --git a/substrate/client/offchain/src/lib.rs b/substrate/client/offchain/src/lib.rs index 756ab77ff94e..8bcfa66a5af3 100644 --- a/substrate/client/offchain/src/lib.rs +++ b/substrate/client/offchain/src/lib.rs @@ -330,7 +330,9 @@ mod tests { use libp2p::{Multiaddr, PeerId}; use sc_block_builder::BlockBuilderBuilder; use sc_client_api::Backend as _; - use sc_network::{config::MultiaddrWithPeerId, types::ProtocolName, ReputationChange}; + use sc_network::{ + config::MultiaddrWithPeerId, types::ProtocolName, ObservedRole, ReputationChange, + }; use sc_transaction_pool::BasicPool; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool}; use sp_consensus::BlockOrigin; @@ -423,6 +425,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } #[test] diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index 25f998385ba1..d078f44f198f 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -753,6 +753,11 @@ where } let protocol_id = config.protocol_id(); + let genesis_hash = client + .block_hash(0u32.into()) + .ok() + .flatten() + .expect("Genesis block exists; qed"); let block_announce_validator = if let Some(f) = block_announce_validator_builder { f(client.clone()) @@ -802,11 +807,7 @@ where // Allow both outgoing and incoming requests. let (handler, protocol_config) = WarpSyncRequestHandler::new( protocol_id.clone(), - client - .block_hash(0u32.into()) - .ok() - .flatten() - .expect("Genesis block exists; qed"), + genesis_hash, config.chain_spec.fork_id(), warp_with_provider.clone(), ); @@ -845,17 +846,13 @@ where } // create transactions protocol and add it to the list of supported protocols of - // `network_params` - let transactions_handler_proto = sc_network_transactions::TransactionsHandlerPrototype::new( - protocol_id.clone(), - client - .block_hash(0u32.into()) - .ok() - .flatten() - .expect("Genesis block exists; qed"), - config.chain_spec.fork_id(), - ); - net_config.add_notification_protocol(transactions_handler_proto.set_config()); + let (transactions_handler_proto, transactions_config) = + sc_network_transactions::TransactionsHandlerPrototype::new( + protocol_id.clone(), + genesis_hash, + config.chain_spec.fork_id(), + ); + net_config.add_notification_protocol(transactions_config); // Create `PeerStore` and initialize it with bootnode peer ids. let peer_store = PeerStore::new( @@ -869,7 +866,6 @@ where let peer_store_handle = peer_store.handle(); spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config.role), client.clone(), @@ -884,7 +880,7 @@ where block_downloader, state_request_protocol_name, warp_request_protocol_name, - rx, + peer_store_handle.clone(), )?; let sync_service_import_queue = sync_service.clone(); let sync_service = Arc::new(sync_service); @@ -905,7 +901,6 @@ where fork_id: config.chain_spec.fork_id().map(ToOwned::to_owned), metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()), block_announce_config, - tx, }; let has_bootnodes = !network_params.network_config.network_config.boot_nodes.is_empty(); diff --git a/substrate/frame/contracts/fixtures/data/account_reentrance_count_call.wat b/substrate/frame/contracts/fixtures/data/account_reentrance_count_call.wat index ab6789066487..e6d6ba8bb814 100644 --- a/substrate/frame/contracts/fixtures/data/account_reentrance_count_call.wat +++ b/substrate/frame/contracts/fixtures/data/account_reentrance_count_call.wat @@ -14,7 +14,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/add_remove_delegate_dependency.wat b/substrate/frame/contracts/fixtures/data/add_remove_delegate_dependency.wat index ef456b6d620a..dac7736244da 100644 --- a/substrate/frame/contracts/fixtures/data/add_remove_delegate_dependency.wat +++ b/substrate/frame/contracts/fixtures/data/add_remove_delegate_dependency.wat @@ -16,7 +16,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -44,8 +44,8 @@ ;; [0..4) - size of the call ;; [4..8) - action to perform ;; [8..42) - code hash of the callee - (set_local $action (i32.load (i32.const 4))) - (set_local $code_hash_ptr (i32.const 8)) + (local.set $action (i32.load (i32.const 4))) + (local.set $code_hash_ptr (i32.const 8)) ;; Assert input size == 36 (4 for action + 32 for code_hash). (call $assert @@ -56,25 +56,25 @@ ) ;; Call add_delegate_dependency when action == 1. - (if (i32.eq (get_local $action) (i32.const 1)) + (if (i32.eq (local.get $action) (i32.const 1)) (then - (call $add_delegate_dependency (get_local $code_hash_ptr)) + (call $add_delegate_dependency (local.get $code_hash_ptr)) ) (else) ) ;; Call remove_delegate_dependency when action == 2. - (if (i32.eq (get_local $action) (i32.const 2)) + (if (i32.eq (local.get $action) (i32.const 2)) (then (call $remove_delegate_dependency - (get_local $code_hash_ptr) + (local.get $code_hash_ptr) ) ) (else) ) ;; Call terminate when action == 3. - (if (i32.eq (get_local $action) (i32.const 3)) + (if (i32.eq (local.get $action) (i32.const 3)) (then (call $terminate (i32.const 100) ;; Pointer to beneficiary address diff --git a/substrate/frame/contracts/fixtures/data/balance.wat b/substrate/frame/contracts/fixtures/data/balance.wat index d86d5c4b1c60..d7970c92e414 100644 --- a/substrate/frame/contracts/fixtures/data/balance.wat +++ b/substrate/frame/contracts/fixtures/data/balance.wat @@ -12,7 +12,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/call.wat b/substrate/frame/contracts/fixtures/data/call.wat index 4558b2c6409b..43b32049c88a 100644 --- a/substrate/frame/contracts/fixtures/data/call.wat +++ b/substrate/frame/contracts/fixtures/data/call.wat @@ -7,7 +7,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/call_runtime_and_call.wat b/substrate/frame/contracts/fixtures/data/call_runtime_and_call.wat index 3320922d9e2c..5d76e19a74c7 100644 --- a/substrate/frame/contracts/fixtures/data/call_runtime_and_call.wat +++ b/substrate/frame/contracts/fixtures/data/call_runtime_and_call.wat @@ -7,7 +7,7 @@ (func $assert (param i32) (block $ok - (br_if $ok (get_local 0)) + (br_if $ok (local.get 0)) (unreachable) ) ) diff --git a/substrate/frame/contracts/fixtures/data/caller_contract.wat b/substrate/frame/contracts/fixtures/data/caller_contract.wat index 929171b9a26f..43eb8ccfd54f 100644 --- a/substrate/frame/contracts/fixtures/data/caller_contract.wat +++ b/substrate/frame/contracts/fixtures/data/caller_contract.wat @@ -10,7 +10,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -37,10 +37,10 @@ ) ;; Read current balance into local variable. - (set_local $sp (i32.const 1024)) + (local.set $sp (i32.const 1024)) ;; Fail to deploy the contract since it returns a non-zero exit status. - (set_local $exit_code + (local.set $exit_code (call $seal_instantiate (i32.const 24) ;; Pointer to the code hash. (i64.const 0) ;; How much ref_time weight to devote for the execution. 0 = all. @@ -60,11 +60,11 @@ ;; Check non-zero exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 2)) ;; ReturnCode::CalleeReverted + (i32.eq (local.get $exit_code) (i32.const 2)) ;; ReturnCode::CalleeReverted ) ;; Fail to deploy the contract due to insufficient ref_time weight. - (set_local $exit_code + (local.set $exit_code (call $seal_instantiate (i32.const 24) ;; Pointer to the code hash. (i64.const 1) ;; Supply too little ref_time weight @@ -85,11 +85,11 @@ ;; Check for special trap exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 1)) ;; ReturnCode::CalleeTrapped + (i32.eq (local.get $exit_code) (i32.const 1)) ;; ReturnCode::CalleeTrapped ) ;; Fail to deploy the contract due to insufficient ref_time weight. - (set_local $exit_code + (local.set $exit_code (call $seal_instantiate (i32.const 24) ;; Pointer to the code hash. (i64.const 0) ;; How much ref_time weight to devote for the execution. 0 = all. @@ -110,17 +110,17 @@ ;; Check for special trap exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 1)) ;; ReturnCode::CalleeTrapped + (i32.eq (local.get $exit_code) (i32.const 1)) ;; ReturnCode::CalleeTrapped ) ;; Length of the output buffer (i32.store - (i32.sub (get_local $sp) (i32.const 4)) + (i32.sub (local.get $sp) (i32.const 4)) (i32.const 256) ) ;; Deploy the contract successfully. - (set_local $exit_code + (local.set $exit_code (call $seal_instantiate (i32.const 24) ;; Pointer to the code hash. (i64.const 0) ;; How much ref_time weight to devote for the execution. 0 = all. @@ -130,7 +130,7 @@ (i32.const 8) ;; Pointer to input data buffer address (i32.const 8) ;; Length of input data buffer (i32.const 16) ;; Pointer to the address output buffer - (i32.sub (get_local $sp) (i32.const 4)) ;; Pointer to the address buffer length + (i32.sub (local.get $sp) (i32.const 4)) ;; Pointer to the address buffer length (i32.const 4294967295) ;; u32 max sentinel value: do not copy output (i32.const 0) ;; Length is ignored in this case (i32.const 0) ;; salt_ptr @@ -141,28 +141,28 @@ ;; Check for success exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 0)) ;; ReturnCode::Success + (i32.eq (local.get $exit_code) (i32.const 0)) ;; ReturnCode::Success ) ;; Check that address has the expected length (call $assert - (i32.eq (i32.load (i32.sub (get_local $sp) (i32.const 4))) (i32.const 32)) + (i32.eq (i32.load (i32.sub (local.get $sp) (i32.const 4))) (i32.const 32)) ) ;; Zero out destination buffer of output (i32.store - (i32.sub (get_local $sp) (i32.const 4)) + (i32.sub (local.get $sp) (i32.const 4)) (i32.const 0) ) ;; Length of the output buffer (i32.store - (i32.sub (get_local $sp) (i32.const 8)) + (i32.sub (local.get $sp) (i32.const 8)) (i32.const 4) ) ;; Call the new contract and expect it to return failing exit code. - (set_local $exit_code + (local.set $exit_code (call $seal_call (i32.const 0) ;; Set no flag (i32.const 16) ;; Pointer to "callee" address. @@ -172,29 +172,29 @@ (i32.const 0) ;; Pointer to the buffer with value to transfer (i32.const 9) ;; Pointer to input data buffer address (i32.const 7) ;; Length of input data buffer - (i32.sub (get_local $sp) (i32.const 4)) ;; Ptr to output buffer - (i32.sub (get_local $sp) (i32.const 8)) ;; Ptr to output buffer len + (i32.sub (local.get $sp) (i32.const 4)) ;; Ptr to output buffer + (i32.sub (local.get $sp) (i32.const 8)) ;; Ptr to output buffer len ) ) ;; Check non-zero exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 2)) ;; ReturnCode::CalleeReverted + (i32.eq (local.get $exit_code) (i32.const 2)) ;; ReturnCode::CalleeReverted ) ;; Check that output buffer contains the expected return data. (call $assert - (i32.eq (i32.load (i32.sub (get_local $sp) (i32.const 8))) (i32.const 3)) + (i32.eq (i32.load (i32.sub (local.get $sp) (i32.const 8))) (i32.const 3)) ) (call $assert (i32.eq - (i32.load (i32.sub (get_local $sp) (i32.const 4))) + (i32.load (i32.sub (local.get $sp) (i32.const 4))) (i32.const 0x00776655) ) ) ;; Fail to call the contract due to insufficient ref_time weight. - (set_local $exit_code + (local.set $exit_code (call $seal_call (i32.const 0) ;; Set no flag (i32.const 16) ;; Pointer to "callee" address. @@ -211,11 +211,11 @@ ;; Check for special trap exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 1)) ;; ReturnCode::CalleeTrapped + (i32.eq (local.get $exit_code) (i32.const 1)) ;; ReturnCode::CalleeTrapped ) ;; Fail to call the contract due to insufficient proof_size weight. - (set_local $exit_code + (local.set $exit_code (call $seal_call (i32.const 0) ;; Set no flag (i32.const 16) ;; Pointer to "callee" address. @@ -232,23 +232,23 @@ ;; Check for special trap exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 1)) ;; ReturnCode::CalleeTrapped + (i32.eq (local.get $exit_code) (i32.const 1)) ;; ReturnCode::CalleeTrapped ) ;; Zero out destination buffer of output (i32.store - (i32.sub (get_local $sp) (i32.const 4)) + (i32.sub (local.get $sp) (i32.const 4)) (i32.const 0) ) ;; Length of the output buffer (i32.store - (i32.sub (get_local $sp) (i32.const 8)) + (i32.sub (local.get $sp) (i32.const 8)) (i32.const 4) ) ;; Call the contract successfully. - (set_local $exit_code + (local.set $exit_code (call $seal_call (i32.const 0) ;; Set no flag (i32.const 16) ;; Pointer to "callee" address. @@ -258,23 +258,23 @@ (i32.const 0) ;; Pointer to the buffer with value to transfer (i32.const 8) ;; Pointer to input data buffer address (i32.const 8) ;; Length of input data buffer - (i32.sub (get_local $sp) (i32.const 4)) ;; Ptr to output buffer - (i32.sub (get_local $sp) (i32.const 8)) ;; Ptr to output buffer len + (i32.sub (local.get $sp) (i32.const 4)) ;; Ptr to output buffer + (i32.sub (local.get $sp) (i32.const 8)) ;; Ptr to output buffer len ) ) ;; Check for success exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 0)) ;; ReturnCode::Success + (i32.eq (local.get $exit_code) (i32.const 0)) ;; ReturnCode::Success ) ;; Check that the output buffer contains the expected return data. (call $assert - (i32.eq (i32.load (i32.sub (get_local $sp) (i32.const 8))) (i32.const 4)) + (i32.eq (i32.load (i32.sub (local.get $sp) (i32.const 8))) (i32.const 4)) ) (call $assert (i32.eq - (i32.load (i32.sub (get_local $sp) (i32.const 4))) + (i32.load (i32.sub (local.get $sp) (i32.const 4))) (i32.const 0x77665544) ) ) diff --git a/substrate/frame/contracts/fixtures/data/chain_extension.wat b/substrate/frame/contracts/fixtures/data/chain_extension.wat index 670f8e70172e..c24ca286ff8c 100644 --- a/substrate/frame/contracts/fixtures/data/chain_extension.wat +++ b/substrate/frame/contracts/fixtures/data/chain_extension.wat @@ -9,7 +9,7 @@ (func $assert (param i32) (block $ok - (br_if $ok (get_local 0)) + (br_if $ok (local.get 0)) (unreachable) ) ) diff --git a/substrate/frame/contracts/fixtures/data/chain_extension_temp_storage.wat b/substrate/frame/contracts/fixtures/data/chain_extension_temp_storage.wat index b481abb5bc7c..504646df1b0e 100644 --- a/substrate/frame/contracts/fixtures/data/chain_extension_temp_storage.wat +++ b/substrate/frame/contracts/fixtures/data/chain_extension_temp_storage.wat @@ -11,7 +11,7 @@ (func $assert (param i32) (block $ok - (br_if $ok (get_local 0)) + (br_if $ok (local.get 0)) (unreachable) ) ) diff --git a/substrate/frame/contracts/fixtures/data/create_storage_and_call.wat b/substrate/frame/contracts/fixtures/data/create_storage_and_call.wat index 5592e7e96a98..2bff53b638fd 100644 --- a/substrate/frame/contracts/fixtures/data/create_storage_and_call.wat +++ b/substrate/frame/contracts/fixtures/data/create_storage_and_call.wat @@ -8,7 +8,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/create_storage_and_instantiate.wat b/substrate/frame/contracts/fixtures/data/create_storage_and_instantiate.wat index cd7202478437..00c9a657f39f 100644 --- a/substrate/frame/contracts/fixtures/data/create_storage_and_instantiate.wat +++ b/substrate/frame/contracts/fixtures/data/create_storage_and_instantiate.wat @@ -14,7 +14,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/crypto_hashes.wat b/substrate/frame/contracts/fixtures/data/crypto_hashes.wat index c2b4d6b81edb..9d86b02f4192 100644 --- a/substrate/frame/contracts/fixtures/data/crypto_hashes.wat +++ b/substrate/frame/contracts/fixtures/data/crypto_hashes.wat @@ -59,8 +59,10 @@ (call $seal_input (local.get $input_ptr) (local.get $input_len_ptr)) (local.set $chosen_hash_fn (i32.load8_u (local.get $input_ptr))) (if (i32.gt_u (local.get $chosen_hash_fn) (i32.const 7)) - ;; We check that the chosen hash fn identifier is within bounds: [0,7] - (unreachable) + (then + ;; We check that the chosen hash fn identifier is within bounds: [0,7] + (unreachable) + ) ) (local.set $input_ptr (i32.add (local.get $input_ptr) (i32.const 1))) (local.set $input_len (i32.sub (i32.load (local.get $input_len_ptr)) (i32.const 1))) diff --git a/substrate/frame/contracts/fixtures/data/debug_message_invalid_utf8.wat b/substrate/frame/contracts/fixtures/data/debug_message_invalid_utf8.wat index e8c447b42fca..dae0de884189 100644 --- a/substrate/frame/contracts/fixtures/data/debug_message_invalid_utf8.wat +++ b/substrate/frame/contracts/fixtures/data/debug_message_invalid_utf8.wat @@ -8,7 +8,7 @@ (func $assert_eq (param i32 i32) (block $ok (br_if $ok - (i32.eq (get_local 0) (get_local 1)) + (i32.eq (local.get 0) (local.get 1)) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/debug_message_logging_disabled.wat b/substrate/frame/contracts/fixtures/data/debug_message_logging_disabled.wat index fc6ee72df8b0..e9ce20ba42b2 100644 --- a/substrate/frame/contracts/fixtures/data/debug_message_logging_disabled.wat +++ b/substrate/frame/contracts/fixtures/data/debug_message_logging_disabled.wat @@ -8,7 +8,7 @@ (func $assert_eq (param i32 i32) (block $ok (br_if $ok - (i32.eq (get_local 0) (get_local 1)) + (i32.eq (local.get 0) (local.get 1)) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/debug_message_works.wat b/substrate/frame/contracts/fixtures/data/debug_message_works.wat index 61933c232961..44a7b6db1bef 100644 --- a/substrate/frame/contracts/fixtures/data/debug_message_works.wat +++ b/substrate/frame/contracts/fixtures/data/debug_message_works.wat @@ -8,7 +8,7 @@ (func $assert_eq (param i32 i32) (block $ok (br_if $ok - (i32.eq (get_local 0) (get_local 1)) + (i32.eq (local.get 0) (local.get 1)) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/delegate_call.wat b/substrate/frame/contracts/fixtures/data/delegate_call.wat index 7fe422af4551..b8d4f0d47f0f 100644 --- a/substrate/frame/contracts/fixtures/data/delegate_call.wat +++ b/substrate/frame/contracts/fixtures/data/delegate_call.wat @@ -24,7 +24,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -70,7 +70,7 @@ ) ;; Call deployed library contract code. - (set_local $exit_code + (local.set $exit_code (call $seal_delegate_call (i32.const 0) ;; Set no call flags (i32.const 64) ;; Pointer to "callee" code_hash. @@ -83,7 +83,7 @@ ;; Check for success exit status. (call $assert - (i32.eq (get_local $exit_code) (i32.const 0)) ;; ReturnCode::Success + (i32.eq (local.get $exit_code) (i32.const 0)) ;; ReturnCode::Success ) (call $assert diff --git a/substrate/frame/contracts/fixtures/data/delegate_call_lib.wat b/substrate/frame/contracts/fixtures/data/delegate_call_lib.wat index 340b9699f875..62eea32800a4 100644 --- a/substrate/frame/contracts/fixtures/data/delegate_call_lib.wat +++ b/substrate/frame/contracts/fixtures/data/delegate_call_lib.wat @@ -20,7 +20,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/delegate_call_simple.wat b/substrate/frame/contracts/fixtures/data/delegate_call_simple.wat index 24ae5a13e33e..ba0a8fcc8ae3 100644 --- a/substrate/frame/contracts/fixtures/data/delegate_call_simple.wat +++ b/substrate/frame/contracts/fixtures/data/delegate_call_simple.wat @@ -12,7 +12,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/destroy_and_transfer.wat b/substrate/frame/contracts/fixtures/data/destroy_and_transfer.wat index 255547955527..2afd3b2fbacf 100644 --- a/substrate/frame/contracts/fixtures/data/destroy_and_transfer.wat +++ b/substrate/frame/contracts/fixtures/data/destroy_and_transfer.wat @@ -33,7 +33,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/drain.wat b/substrate/frame/contracts/fixtures/data/drain.wat index cb8ff0aed61f..18a21cca803d 100644 --- a/substrate/frame/contracts/fixtures/data/drain.wat +++ b/substrate/frame/contracts/fixtures/data/drain.wat @@ -19,7 +19,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/ecdsa_recover.wat b/substrate/frame/contracts/fixtures/data/ecdsa_recover.wat index d694b3215e86..4910e706069e 100644 --- a/substrate/frame/contracts/fixtures/data/ecdsa_recover.wat +++ b/substrate/frame/contracts/fixtures/data/ecdsa_recover.wat @@ -12,7 +12,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/event_size.wat b/substrate/frame/contracts/fixtures/data/event_size.wat index 4bd6158d72fb..1c1f34b24d72 100644 --- a/substrate/frame/contracts/fixtures/data/event_size.wat +++ b/substrate/frame/contracts/fixtures/data/event_size.wat @@ -9,7 +9,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/multi_store.wat b/substrate/frame/contracts/fixtures/data/multi_store.wat index 2592baf61835..c334ed54c4eb 100644 --- a/substrate/frame/contracts/fixtures/data/multi_store.wat +++ b/substrate/frame/contracts/fixtures/data/multi_store.wat @@ -19,7 +19,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/reentrance_count_call.wat b/substrate/frame/contracts/fixtures/data/reentrance_count_call.wat index c6b529e2aff8..44db8d041b1d 100644 --- a/substrate/frame/contracts/fixtures/data/reentrance_count_call.wat +++ b/substrate/frame/contracts/fixtures/data/reentrance_count_call.wat @@ -20,7 +20,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -36,11 +36,11 @@ (call $seal_input (i32.const 32) (i32.const 36)) ;; reading manually passed reentrant count - (set_local $expected_reentrance_count (i32.load (i32.const 32))) + (local.set $expected_reentrance_count (i32.load (i32.const 32))) ;; reentrance count is calculated correctly (call $assert - (i32.eq (call $reentrance_count) (get_local $expected_reentrance_count)) + (i32.eq (call $reentrance_count) (local.get $expected_reentrance_count)) ) ;; re-enter 5 times in a row and assert that the reentrant counter works as expected @@ -52,7 +52,7 @@ (i32.store (i32.const 32) (i32.add (i32.load (i32.const 32)) (i32.const 1))) ;; Call to itself - (set_local $seal_call_exit_code + (local.set $seal_call_exit_code (call $seal_call (i32.const 8) ;; Allow reentrancy flag set (i32.const 0) ;; Pointer to "callee" address @@ -66,7 +66,7 @@ ) (call $assert - (i32.eq (get_local $seal_call_exit_code) (i32.const 0)) + (i32.eq (local.get $seal_call_exit_code) (i32.const 0)) ) ) ) diff --git a/substrate/frame/contracts/fixtures/data/reentrance_count_delegated_call.wat b/substrate/frame/contracts/fixtures/data/reentrance_count_delegated_call.wat index b8219a8462ee..49e0193bcdb1 100644 --- a/substrate/frame/contracts/fixtures/data/reentrance_count_delegated_call.wat +++ b/substrate/frame/contracts/fixtures/data/reentrance_count_delegated_call.wat @@ -17,7 +17,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -30,7 +30,7 @@ (call $seal_input (i32.const 0) (i32.const 36)) ;; reading passed callstack height - (set_local $callstack_height (i32.load (i32.const 32))) + (local.set $callstack_height (i32.load (i32.const 32))) ;; incrementing callstack height (i32.store (i32.const 32) (i32.add (i32.load (i32.const 32)) (i32.const 1))) @@ -40,12 +40,12 @@ (i32.eq (call $reentrance_count) (i32.const 0)) ) - (i32.eq (get_local $callstack_height) (i32.const 5)) + (i32.eq (local.get $callstack_height) (i32.const 5)) (if (then) ;; exit recursion case (else ;; Call to itself - (set_local $delegate_call_exit_code + (local.set $delegate_call_exit_code (call $seal_delegate_call (i32.const 0) ;; Set no call flags (i32.const 0) ;; Pointer to "callee" code_hash. @@ -57,13 +57,13 @@ ) (call $assert - (i32.eq (get_local $delegate_call_exit_code) (i32.const 0)) + (i32.eq (local.get $delegate_call_exit_code) (i32.const 0)) ) ) ) (call $assert - (i32.le_s (get_local $callstack_height) (i32.const 5)) + (i32.le_s (local.get $callstack_height) (i32.const 5)) ) ) diff --git a/substrate/frame/contracts/fixtures/data/self_destruct.wat b/substrate/frame/contracts/fixtures/data/self_destruct.wat index b8a37306e201..00c3895fdded 100644 --- a/substrate/frame/contracts/fixtures/data/self_destruct.wat +++ b/substrate/frame/contracts/fixtures/data/self_destruct.wat @@ -26,7 +26,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/self_destructing_constructor.wat b/substrate/frame/contracts/fixtures/data/self_destructing_constructor.wat index 85fce511e21b..628f283a19fd 100644 --- a/substrate/frame/contracts/fixtures/data/self_destructing_constructor.wat +++ b/substrate/frame/contracts/fixtures/data/self_destructing_constructor.wat @@ -5,7 +5,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/set_code_hash.wat b/substrate/frame/contracts/fixtures/data/set_code_hash.wat index b4df1b133186..c0a9557b4d00 100644 --- a/substrate/frame/contracts/fixtures/data/set_code_hash.wat +++ b/substrate/frame/contracts/fixtures/data/set_code_hash.wat @@ -16,7 +16,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -27,11 +27,11 @@ (call $seal_input (i32.const 0) (i32.const 32)) - (set_local $exit_code + (local.set $exit_code (call $seal_set_code_hash (i32.const 0)) ;; Pointer to the input data. ) (call $assert - (i32.eq (get_local $exit_code) (i32.const 0)) ;; ReturnCode::Success + (i32.eq (local.get $exit_code) (i32.const 0)) ;; ReturnCode::Success ) ;; we return 1 after setting new code_hash diff --git a/substrate/frame/contracts/fixtures/data/storage_size.wat b/substrate/frame/contracts/fixtures/data/storage_size.wat index 293a656d4f6e..728bb4fcf3c0 100644 --- a/substrate/frame/contracts/fixtures/data/storage_size.wat +++ b/substrate/frame/contracts/fixtures/data/storage_size.wat @@ -20,7 +20,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/store_call.wat b/substrate/frame/contracts/fixtures/data/store_call.wat index 9e090d31801f..746b7a48b551 100644 --- a/substrate/frame/contracts/fixtures/data/store_call.wat +++ b/substrate/frame/contracts/fixtures/data/store_call.wat @@ -15,7 +15,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/store_deploy.wat b/substrate/frame/contracts/fixtures/data/store_deploy.wat index cc428e9623bf..7f115cba977c 100644 --- a/substrate/frame/contracts/fixtures/data/store_deploy.wat +++ b/substrate/frame/contracts/fixtures/data/store_deploy.wat @@ -15,7 +15,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/xcm_execute.wat b/substrate/frame/contracts/fixtures/data/xcm_execute.wat index b3459996a2e8..72ef14ed82c7 100644 --- a/substrate/frame/contracts/fixtures/data/xcm_execute.wat +++ b/substrate/frame/contracts/fixtures/data/xcm_execute.wat @@ -12,7 +12,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/fixtures/data/xcm_send.wat b/substrate/frame/contracts/fixtures/data/xcm_send.wat index 9eec6388de9b..fe29ddf0f141 100644 --- a/substrate/frame/contracts/fixtures/data/xcm_send.wat +++ b/substrate/frame/contracts/fixtures/data/xcm_send.wat @@ -12,7 +12,7 @@ (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) diff --git a/substrate/frame/contracts/src/wasm/mod.rs b/substrate/frame/contracts/src/wasm/mod.rs index 77e94b16777b..f73655e99203 100644 --- a/substrate/frame/contracts/src/wasm/mod.rs +++ b/substrate/frame/contracts/src/wasm/mod.rs @@ -1506,7 +1506,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -1531,7 +1531,7 @@ mod tests { ) ;; Find out the size of the buffer - (set_local $buf_size + (local.set $buf_size (i32.load (i32.const 32)) ) @@ -1539,7 +1539,7 @@ mod tests { (call $seal_return (i32.const 0) (i32.const 36) - (get_local $buf_size) + (local.get $buf_size) ) ;; env:seal_return doesn't return, so this is effectively unreachable. @@ -1575,7 +1575,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -1633,7 +1633,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -1680,7 +1680,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -1726,7 +1726,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -1773,7 +1773,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -1836,7 +1836,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -1925,7 +1925,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -1966,7 +1966,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -2013,7 +2013,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -2067,7 +2067,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -2137,7 +2137,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -2327,7 +2327,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -2995,7 +2995,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -3047,7 +3047,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) @@ -3162,18 +3162,18 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) ) (func (export "call") (local $exit_code i32) - (set_local $exit_code + (local.set $exit_code (call $seal_set_code_hash (i32.const 0)) ) (call $assert - (i32.eq (get_local $exit_code) (i32.const 0)) ;; ReturnCode::Success + (i32.eq (local.get $exit_code) (i32.const 0)) ;; ReturnCode::Success ) ) @@ -3202,18 +3202,18 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) ) (func (export "call") (local $return_val i32) - (set_local $return_val + (local.set $return_val (call $reentrance_count) ) (call $assert - (i32.eq (get_local $return_val) (i32.const 12)) + (i32.eq (local.get $return_val) (i32.const 12)) ) ) @@ -3234,18 +3234,18 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) ) ) (func (export "call") (local $return_val i32) - (set_local $return_val + (local.set $return_val (call $account_reentrance_count (i32.const 0)) ) (call $assert - (i32.eq (get_local $return_val) (i32.const 12)) + (i32.eq (local.get $return_val) (i32.const 12)) ) ) @@ -3267,7 +3267,7 @@ mod tests { (func $assert (param i32) (block $ok (br_if $ok - (get_local 0) + (local.get 0) ) (unreachable) )