diff --git a/core/common/ref_cache.hpp b/core/common/ref_cache.hpp index b72036ca4b..f767bbecd3 100644 --- a/core/common/ref_cache.hpp +++ b/core/common/ref_cache.hpp @@ -9,6 +9,7 @@ #include #include #include +#include "outcome/outcome.hpp" namespace kagome { @@ -91,14 +92,15 @@ namespace kagome { * @param f is a functor to create RefObj */ template - SRefObj get_or_insert(const Key &k, F &&f) { + outcome::result get_or_insert(const Key &k, F &&f) { [[likely]] if (auto it = items_.find(k); it != items_.end()) { auto obj = it->second.lock(); BOOST_ASSERT(obj); return obj; } - auto obj = std::make_shared(std::forward(f)()); + OUTCOME_TRY(o, std::forward(f)()); + auto obj = std::make_shared(std::move(o)); auto [it, inserted] = items_.emplace(k, obj); BOOST_ASSERT(inserted); @@ -107,6 +109,10 @@ namespace kagome { return obj; } + size_t size() const { + return items_.size(); + } + private: RefCache() = default; diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 6805c0687d..6e063f3f57 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -162,6 +162,7 @@ #include "parachain/pvf/workers.hpp" #include "parachain/validator/impl/parachain_observer_impl.hpp" #include "parachain/validator/parachain_processor.hpp" +#include "parachain/validator/statement_distribution/statement_distribution.hpp" #include "runtime/binaryen/binaryen_memory_provider.hpp" #include "runtime/binaryen/instance_environment_factory.hpp" #include "runtime/binaryen/module/module_factory_impl.hpp" @@ -788,7 +789,7 @@ namespace { di::bind.template to(), di::bind.template to(), di::bind.template to(), - di::bind.template to(), + di::bind.template to(), di::bind.template to(), di::bind.template to(), di::bind.template to(), @@ -1011,6 +1012,12 @@ namespace kagome::injector { .template create>(); } + std::shared_ptr + KagomeNodeInjector::injectStatementDistribution() { + return pimpl_->injector_.template create< + sptr>(); + } + std::shared_ptr KagomeNodeInjector::injectApprovalDistribution() { return pimpl_->injector_ diff --git a/core/injector/application_injector.hpp b/core/injector/application_injector.hpp index db15bf9302..89ae6faaf6 100644 --- a/core/injector/application_injector.hpp +++ b/core/injector/application_injector.hpp @@ -60,6 +60,10 @@ namespace kagome { class ParachainObserver; struct ParachainProcessorImpl; struct ApprovalDistribution; + + namespace statement_distribution { + struct StatementDistribution; + } } // namespace parachain namespace runtime { @@ -131,6 +135,8 @@ namespace kagome::injector { std::shared_ptr injectParachainObserver(); std::shared_ptr injectParachainProcessor(); + std::shared_ptr + injectStatementDistribution(); std::shared_ptr injectApprovalDistribution(); std::shared_ptr diff --git a/core/network/can_disconnect.hpp b/core/network/can_disconnect.hpp index dcbddde136..02bf5c05b6 100644 --- a/core/network/can_disconnect.hpp +++ b/core/network/can_disconnect.hpp @@ -18,6 +18,6 @@ namespace kagome::network { public: virtual ~CanDisconnect() = default; - virtual bool canDisconnect(const libp2p::PeerId &) const = 0; + virtual bool can_disconnect(const libp2p::PeerId &) const = 0; }; } // namespace kagome::network diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index a7f12b9534..b7b5e9b6bd 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -93,7 +93,8 @@ namespace kagome::network { std::shared_ptr hasher, std::shared_ptr reputation_repository, LazySPtr can_disconnect, - std::shared_ptr peer_view) + std::shared_ptr peer_view, + primitives::events::PeerSubscriptionEnginePtr peer_event_engine) : log_{log::createLogger("PeerManager", "network")}, host_(host), main_pool_handler_{poolHandlerReadyMake( @@ -111,7 +112,8 @@ namespace kagome::network { hasher_{std::move(hasher)}, reputation_repository_{std::move(reputation_repository)}, can_disconnect_{can_disconnect}, - peer_view_{std::move(peer_view)} { + peer_view_{std::move(peer_view)}, + peer_event_engine_(std::move(peer_event_engine)) { BOOST_ASSERT(identify_ != nullptr); BOOST_ASSERT(kademlia_ != nullptr); BOOST_ASSERT(scheduler_ != nullptr); @@ -122,6 +124,7 @@ namespace kagome::network { BOOST_ASSERT(peer_view_); BOOST_ASSERT(reputation_repository_ != nullptr); BOOST_ASSERT(peer_view_ != nullptr); + BOOST_ASSERT(peer_event_engine_); // Register metrics registry_->registerGaugeFamily(syncPeerMetricName, @@ -188,6 +191,9 @@ namespace kagome::network { self->peer_view_->removePeer(peer_id); self->sync_peer_num_->set(self->active_peers_.size()); self->peers_count_metric_->set(self->active_peers_.size()); + + self->peer_event_engine_->notify( + primitives::events::PeerEventType::kDisconnected, peer_id); SL_DEBUG(self->log_, "Remained {} active peers", self->active_peers_.size()); @@ -331,7 +337,7 @@ namespace kagome::network { for (const auto &[peer_id, desc] : active_peers_) { // Skip peer having immunity - if (not can_disconnect_.get()->canDisconnect(peer_id)) { + if (not can_disconnect_.get()->can_disconnect(peer_id)) { continue; } @@ -858,11 +864,9 @@ namespace kagome::network { } } + self->peer_event_engine_->notify( + primitives::events::PeerEventType::kConnected, peer_info.id); self->tryOpenGrandpaProtocol(peer_info, peer_state.value().get()); - self->tryOpenValidationProtocol( - peer_info, - peer_state.value().get(), - network::CollationVersion::VStaging); auto beefy_protocol = std::static_pointer_cast( self->router_->getBeefyProtocol()); openOutgoing( diff --git a/core/network/impl/peer_manager_impl.hpp b/core/network/impl/peer_manager_impl.hpp index 72ee0d7f22..c690677809 100644 --- a/core/network/impl/peer_manager_impl.hpp +++ b/core/network/impl/peer_manager_impl.hpp @@ -79,7 +79,8 @@ namespace kagome::network { std::shared_ptr hasher, std::shared_ptr reputation_repository, LazySPtr can_disconnect, - std::shared_ptr peer_view); + std::shared_ptr peer_view, + primitives::events::PeerSubscriptionEnginePtr peer_event_engine); /** @see poolHandlerReadyMake */ bool tryStart(); @@ -222,6 +223,7 @@ namespace kagome::network { std::unordered_map peer_states_; libp2p::basic::Scheduler::Handle align_timer_; std::set recently_active_peers_; + primitives::events::PeerSubscriptionEnginePtr peer_event_engine_; // metrics metrics::RegistryPtr registry_ = metrics::createRegistry(); diff --git a/core/network/impl/protocols/fetch_attested_candidate.hpp b/core/network/impl/protocols/fetch_attested_candidate.hpp index fcdeafe7d4..1f4a72f2f7 100644 --- a/core/network/impl/protocols/fetch_attested_candidate.hpp +++ b/core/network/impl/protocols/fetch_attested_candidate.hpp @@ -19,6 +19,7 @@ #include "network/impl/protocols/request_response_protocol.hpp" #include "network/impl/stream_engine.hpp" #include "parachain/validator/parachain_processor.hpp" +#include "parachain/validator/statement_distribution/statement_distribution.hpp" #include "utils/non_copyable.hpp" namespace kagome::network { @@ -34,7 +35,9 @@ namespace kagome::network { libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr pp) + std::shared_ptr< + parachain::statement_distribution::StatementDistribution> + statement_distribution) : RequestResponseProtocolImpl< vstaging::AttestedCandidateRequest, vstaging::AttestedCandidateResponse, @@ -47,8 +50,8 @@ namespace kagome::network { log::createLogger( kFetchAttestedCandidateProtocolName, "req_attested_candidate_protocol")}, - pp_{std::move(pp)} { - BOOST_ASSERT(pp_); + statement_distribution_(std::move(statement_distribution)) { + BOOST_ASSERT(statement_distribution_); } private: @@ -57,16 +60,8 @@ namespace kagome::network { base().logger()->info( "Fetching attested candidate request.(candidate={})", request.candidate_hash); - auto res = pp_->OnFetchAttestedCandidateRequest( - std::move(request), stream->remotePeerId().value()); - if (res.has_error()) { - base().logger()->error( - "Fetching attested candidate response failed.(error={})", - res.error()); - } else { - SL_TRACE(base().logger(), "Fetching attested candidate response."); - } - return res; + statement_distribution_->OnFetchAttestedCandidateRequest(request, stream); + return std::nullopt; } void onTxRequest(const RequestType &request) override { @@ -76,7 +71,8 @@ namespace kagome::network { inline static const auto kFetchAttestedCandidateProtocolName = "FetchAttestedCandidateProtocol"s; - std::shared_ptr pp_; + std::shared_ptr + statement_distribution_; }; } // namespace kagome::network diff --git a/core/network/impl/protocols/request_response_protocol.hpp b/core/network/impl/protocols/request_response_protocol.hpp index c7a2e5c3f1..326b8e3b32 100644 --- a/core/network/impl/protocols/request_response_protocol.hpp +++ b/core/network/impl/protocols/request_response_protocol.hpp @@ -86,6 +86,11 @@ namespace kagome::network { }); } + void writeResponseAsync(std::shared_ptr stream, + ResponseType response) { + writeResponse(std::move(stream), std::move(response)); + } + protected: virtual std::optional> onRxRequest( RequestType request, std::shared_ptr stream) = 0; diff --git a/core/network/peer_state.hpp b/core/network/peer_state.hpp index 13f23bea91..fdb0782dfc 100644 --- a/core/network/peer_state.hpp +++ b/core/network/peer_state.hpp @@ -60,66 +60,9 @@ namespace kagome::network { /// @brief parachain peer state std::optional collator_state = std::nullopt; - View view; - std::unordered_set implicit_view; std::optional collation_version; std::optional req_chunk_version; - /// Update the view, returning a vector of implicit relay-parents which - /// weren't previously part of the view. - std::vector update_view( - const View &new_view, const parachain::ImplicitView &local_implicit) { - std::unordered_set next_implicit; - for (const auto &x : new_view.heads_) { - auto t = - local_implicit.known_allowed_relay_parents_under(x, std::nullopt); - if (t) { - next_implicit.insert(t->begin(), t->end()); - } - } - - std::vector fresh_implicit; - for (const auto &x : next_implicit) { - if (implicit_view.find(x) == implicit_view.end()) { - fresh_implicit.emplace_back(x); - } - } - - view = new_view; - implicit_view = next_implicit; - return fresh_implicit; - } - - /// Whether we know that a peer knows a relay-parent. The peer knows the - /// relay-parent if it is either implicit or explicit in their view. - /// However, if it is implicit via an active-leaf we don't recognize, we - /// will not accurately be able to recognize them as 'knowing' the - /// relay-parent. - bool knows_relay_parent(const common::Hash256 &relay_parent) { - return implicit_view.contains(relay_parent) - || view.contains(relay_parent); - } - - /// Attempt to reconcile the view with new information about the implicit - /// relay parents under an active leaf. - std::vector reconcile_active_leaf( - const common::Hash256 &leaf_hash, - std::span implicit) { - if (!view.contains(leaf_hash)) { - return {}; - } - - std::vector v; - v.reserve(implicit.size()); - for (const auto &i : implicit) { - auto [_, inserted] = implicit_view.insert(i); - if (inserted) { - v.emplace_back(i); - } - } - return v; - } - /// Whether the peer has advertised the given collation. bool hasAdvertised( const RelayHash &relay_parent, diff --git a/core/network/types/collator_messages.hpp b/core/network/types/collator_messages.hpp index f917bab494..792457ea8b 100644 --- a/core/network/types/collator_messages.hpp +++ b/core/network/types/collator_messages.hpp @@ -88,41 +88,6 @@ namespace kagome::network { using RequestPov = CandidateHash; using ResponsePov = boost::variant; - /** - * Contains information about the candidate and a proof of the results of its - * execution. - */ - struct CandidateReceipt { - CandidateDescriptor descriptor; /// Candidate descriptor - Hash commitments_hash; /// Hash of candidate commitments - - const Hash &hash(const crypto::Hasher &hasher) const { - if (not hash_.has_value()) { - hash_.emplace(hasher.blake2b_256( - ::scale::encode(std::tie(descriptor, commitments_hash)).value())); - } - return hash_.value(); - } - - inline bool operator==(const CandidateReceipt &other) const { - return descriptor == other.descriptor - and commitments_hash == other.commitments_hash; - } - - friend inline scale::ScaleDecoderStream &operator>>( - scale::ScaleDecoderStream &s, CandidateReceipt &cr) { - return s >> cr.descriptor >> cr.commitments_hash; - } - - friend inline scale::ScaleEncoderStream &operator<<( - scale::ScaleEncoderStream &s, const CandidateReceipt &cr) { - return s << cr.descriptor << cr.commitments_hash; - } - - private: - mutable std::optional hash_{}; - }; - struct CollationResponse { SCALE_TIE(2); @@ -212,23 +177,6 @@ namespace kagome::network { using FetchAvailableDataResponse = boost::variant; - struct CommittedCandidateReceipt { - SCALE_TIE(2); - - CandidateDescriptor descriptor; /// Candidate descriptor - CandidateCommitments commitments; /// commitments retrieved from validation - /// result and produced by the execution - /// and validation parachain candidate - - CandidateReceipt to_plain(const crypto::Hasher &hasher) const { - CandidateReceipt receipt; - receipt.descriptor = descriptor, - receipt.commitments_hash = - hasher.blake2b_256(scale::encode(commitments).value()); - return receipt; - } - }; - struct FetchStatementRequest { SCALE_TIE(2); RelayHash relay_parent; diff --git a/core/parachain/CMakeLists.txt b/core/parachain/CMakeLists.txt index 0a2fa031b4..d36c491079 100644 --- a/core/parachain/CMakeLists.txt +++ b/core/parachain/CMakeLists.txt @@ -41,6 +41,7 @@ add_library(validator_parachain backing/store_impl.cpp backing/cluster.cpp validator/backing_implicit_view.cpp + validator/statement_distribution/statement_distribution.cpp ) target_link_libraries(validator_parachain diff --git a/core/parachain/approval/approval_thread_pool.hpp b/core/parachain/approval/approval_thread_pool.hpp index c9d82fc338..711afe0e56 100644 --- a/core/parachain/approval/approval_thread_pool.hpp +++ b/core/parachain/approval/approval_thread_pool.hpp @@ -15,4 +15,11 @@ namespace kagome::parachain { ApprovalThreadPool(std::shared_ptr watchdog) : ThreadPool(std::move(watchdog), "approval", 1, std::nullopt) {} }; + + class StatementDistributionThreadPool final : public ThreadPool { + public: + StatementDistributionThreadPool(std::shared_ptr watchdog) + : ThreadPool( + std::move(watchdog), "statement-distribution", 1, std::nullopt) {} + }; } // namespace kagome::parachain diff --git a/core/parachain/availability/availability_chunk_index.hpp b/core/parachain/availability/availability_chunk_index.hpp index 8ed51a021f..d71d9eafaa 100644 --- a/core/parachain/availability/availability_chunk_index.hpp +++ b/core/parachain/availability/availability_chunk_index.hpp @@ -67,7 +67,7 @@ namespace kagome::parachain { /// WARNING: THIS FUNCTION IS CRITICAL TO PARACHAIN CONSENSUS. /// Any modification to the output of the function needs to be coordinated via /// the runtime. It's best to use minimal/no external dependencies. - outcome::result availability_chunk_index( + inline outcome::result availability_chunk_index( std::optional node_features, size_t n_validators, CoreIndex core_index, diff --git a/core/parachain/availability/bitfield/signer.cpp b/core/parachain/availability/bitfield/signer.cpp index a558823a78..f138065aed 100644 --- a/core/parachain/availability/bitfield/signer.cpp +++ b/core/parachain/availability/bitfield/signer.cpp @@ -7,6 +7,7 @@ #include "parachain/availability/bitfield/signer.hpp" #include "log/logger.hpp" +#include "parachain/availability/availability_chunk_index.hpp" #include "primitives/block_header.hpp" namespace kagome::parachain { @@ -78,11 +79,30 @@ namespace kagome::parachain { OUTCOME_TRY( session, parachain_api_->session_info(relay_parent, signer->getSessionIndex())); + OUTCOME_TRY( + node_features, + parachain_api_->node_features(relay_parent, signer->getSessionIndex())); candidates.reserve(cores.size()); - for (auto &core : cores) { + for (CoreIndex core_index = 0; core_index < cores.size(); ++core_index) { + auto &core = cores[core_index]; if (auto occupied = std::get_if(&core)) { candidates.emplace_back(occupied->candidate_hash); - fetch_->fetch(signer->validatorIndex(), *occupied, *session); + + size_t n_validators = 0; + for (const auto &group : session->validator_groups) { + n_validators += group.size(); + } + + SL_DEBUG(logger_, + "chunk mapping is enabled: {}", + availability_chunk_mapping_is_enabled(node_features) ? "YES" + : "NO"); + OUTCOME_TRY(chunk_index, + availability_chunk_index(node_features, + n_validators, + core_index, + signer->validatorIndex())); + fetch_->fetch(chunk_index, *occupied, *session); } else { candidates.emplace_back(std::nullopt); } diff --git a/core/parachain/availability/fetch/fetch_impl.cpp b/core/parachain/availability/fetch/fetch_impl.cpp index 1b884ff897..f70297233c 100644 --- a/core/parachain/availability/fetch/fetch_impl.cpp +++ b/core/parachain/availability/fetch/fetch_impl.cpp @@ -43,6 +43,7 @@ namespace kagome::parachain { if (av_store_->hasChunk(core.candidate_hash, chunk_index)) { return; } + Active active; active.chunk_index = chunk_index; active.relay_parent = core.candidate_descriptor.relay_parent; diff --git a/core/parachain/availability/store/store_impl.cpp b/core/parachain/availability/store/store_impl.cpp index 1bf6d61946..28d20a6c6a 100644 --- a/core/parachain/availability/store/store_impl.cpp +++ b/core/parachain/availability/store/store_impl.cpp @@ -6,7 +6,12 @@ #include "parachain/availability/store/store_impl.hpp" +constexpr uint64_t KEEP_CANDIDATES_TIMEOUT = 10 * 60; + namespace kagome::parachain { + AvailabilityStoreImpl::AvailabilityStoreImpl(clock::SteadyClock &steady_clock) + : steady_clock_(steady_clock) {} + bool AvailabilityStoreImpl::hasChunk(const CandidateHash &candidate_hash, ValidatorIndex index) const { return state_.sharedAccess([&](const auto &state) { @@ -113,12 +118,23 @@ namespace kagome::parachain { }); } + void AvailabilityStoreImpl::prune_candidates_no_lock(State &state) { + const auto now = steady_clock_.nowUint64(); + while (!state.candidates_living_keeper_.empty() + && state.candidates_living_keeper_[0].first + KEEP_CANDIDATES_TIMEOUT + < now) { + remove_no_lock(state, state.candidates_living_keeper_[0].second); + state.candidates_living_keeper_.pop_front(); + } + } + void AvailabilityStoreImpl::storeData(const network::RelayHash &relay_parent, const CandidateHash &candidate_hash, std::vector &&chunks, const ParachainBlock &pov, const PersistedValidationData &data) { state_.exclusiveAccess([&](auto &state) { + prune_candidates_no_lock(state); state.candidates_[relay_parent].insert(candidate_hash); auto &candidate_data = state.per_candidate_[candidate_hash]; for (auto &&chunk : std::move(chunks)) { @@ -126,6 +142,8 @@ namespace kagome::parachain { } candidate_data.pov = pov; candidate_data.data = data; + state.candidates_living_keeper_.emplace_back(steady_clock_.nowUint64(), + relay_parent); }); } @@ -133,21 +151,28 @@ namespace kagome::parachain { const CandidateHash &candidate_hash, ErasureChunk &&chunk) { state_.exclusiveAccess([&](auto &state) { + prune_candidates_no_lock(state); state.candidates_[relay_parent].insert(candidate_hash); state.per_candidate_[candidate_hash].chunks[chunk.index] = std::move(chunk); + state.candidates_living_keeper_.emplace_back(steady_clock_.nowUint64(), + relay_parent); }); } - void AvailabilityStoreImpl::remove(const network::RelayHash &relay_parent) { - state_.exclusiveAccess([&](auto &state) { - if (auto it = state.candidates_.find(relay_parent); - it != state.candidates_.end()) { - for (auto const &l : it->second) { - state.per_candidate_.erase(l); - } - state.candidates_.erase(it); + void AvailabilityStoreImpl::remove_no_lock( + State &state, const network::RelayHash &relay_parent) { + if (auto it = state.candidates_.find(relay_parent); + it != state.candidates_.end()) { + for (const auto &l : it->second) { + state.per_candidate_.erase(l); } - }); + state.candidates_.erase(it); + } + } + + void AvailabilityStoreImpl::remove(const network::RelayHash &relay_parent) { + state_.exclusiveAccess( + [&](auto &state) { remove_no_lock(state, relay_parent); }); } } // namespace kagome::parachain diff --git a/core/parachain/availability/store/store_impl.hpp b/core/parachain/availability/store/store_impl.hpp index 861aa2bc88..7d77074dfd 100644 --- a/core/parachain/availability/store/store_impl.hpp +++ b/core/parachain/availability/store/store_impl.hpp @@ -16,6 +16,7 @@ namespace kagome::parachain { class AvailabilityStoreImpl : public AvailabilityStore { public: + AvailabilityStoreImpl(clock::SteadyClock &steady_clock); ~AvailabilityStoreImpl() override = default; bool hasChunk(const CandidateHash &candidate_hash, @@ -52,9 +53,15 @@ namespace kagome::parachain { std::unordered_map per_candidate_{}; std::unordered_map> candidates_{}; + std::deque> + candidates_living_keeper_; }; + void prune_candidates_no_lock(State &state); + void remove_no_lock(State &state, const network::RelayHash &relay_parent); + log::Logger logger = log::createLogger("AvailabilityStore", "parachain"); + clock::SteadyClock &steady_clock_; SafeObject state_{}; }; } // namespace kagome::parachain diff --git a/core/parachain/backing/grid_tracker.cpp b/core/parachain/backing/grid_tracker.cpp index c76db7200f..32f501413f 100644 --- a/core/parachain/backing/grid_tracker.cpp +++ b/core/parachain/backing/grid_tracker.cpp @@ -147,10 +147,36 @@ namespace kagome::parachain::grid { bool manifest_allowed = true; if (kind == ManifestKind::Full) { manifest_allowed = receiving_from; + SL_TRACE(logger, + "Manifest full allowed. (receiving_from={})", + manifest_allowed ? "[yes]" : "[no]"); } else { auto it = confirmed_backed.find(candidate_hash); manifest_allowed = sending_to && it != confirmed_backed.end() && it->second.has_sent_manifest_to(sender); + SL_TRACE(logger, + "Manifest acknowledgement allowed. (sender={}, " + "candidate_hash={}, sending_to={}, " + "has_confirmed_back={}, has_sent_manifest_to={})", + sender, + candidate_hash, + sending_to ? "[yes]" : "[no]", + (it != confirmed_backed.end()) ? "[yes]" : "[no]", + (it != confirmed_backed.end() + && it->second.has_sent_manifest_to(sender)) + ? "[yes]" + : "[no]"); + if (it != confirmed_backed.end()) { + auto a = it->second.mutual_knowledge.find(sender); + SL_TRACE( + logger, + "Local knowledge. (has_mutual_knowledge={}, local_knowledge={})", + (a != it->second.mutual_knowledge.end()) ? "[yes]" : "[no]", + (a != it->second.mutual_knowledge.end()) + && a->second.local_knowledge + ? "[yes]" + : "[no]"); + } } if (!manifest_allowed) { return Error::DISALLOWED_DIRECTION; @@ -266,6 +292,11 @@ namespace kagome::parachain::grid { const StatementFilter &local_knowledge) { auto confirmed = confirmed_backed.find(candidate_hash); if (confirmed != confirmed_backed.end()) { + SL_TRACE(logger, + "Manifest sent to. (validator_index={}, candidate_hash={}, " + "local_knowledge=[yes])", + validator_index, + candidate_hash); confirmed->second.manifest_sent_to(validator_index, local_knowledge); auto ps = confirmed->second.pending_statements(validator_index); if (ps) { @@ -481,9 +512,16 @@ namespace kagome::parachain::grid { return recipients; } for (const auto &[v, k] : mutual_knowledge) { - if (k.local_knowledge - && !k.remote_knowledge->contains(originator_index_in_group, - statement_kind)) { + if (!k.local_knowledge) { + continue; + } + + if (!k.remote_knowledge) { + continue; + } + + if (!k.remote_knowledge->contains(originator_index_in_group, + statement_kind)) { recipients.push_back(v); } } diff --git a/core/parachain/backing/grid_tracker.hpp b/core/parachain/backing/grid_tracker.hpp index 76945f2297..3c6cef8860 100644 --- a/core/parachain/backing/grid_tracker.hpp +++ b/core/parachain/backing/grid_tracker.hpp @@ -16,6 +16,7 @@ #include #include +#include "log/logger.hpp" #include "network/types/collator_messages_vstaging.hpp" #include "parachain/backing/grid.hpp" #include "parachain/groups.hpp" @@ -312,6 +313,7 @@ namespace kagome::parachain::grid { std::unordered_set< std::pair>> pending_statements; + log::Logger logger = log::createLogger("GridTracker", "parachain"); }; } // namespace kagome::parachain::grid diff --git a/core/parachain/types.hpp b/core/parachain/types.hpp index f3843f8b46..eb7e7cb530 100644 --- a/core/parachain/types.hpp +++ b/core/parachain/types.hpp @@ -181,6 +181,44 @@ namespace kagome::network { }; } }; + + /** + * Contains information about the candidate and a proof of the results of its + * execution. + */ + struct CandidateReceipt { + CandidateDescriptor descriptor; /// Candidate descriptor + parachain::Hash commitments_hash; /// Hash of candidate commitments + mutable std::optional hash_{}; + + const parachain::Hash &hash(const crypto::Hasher &hasher) const { + if (not hash_.has_value()) { + hash_.emplace(hasher.blake2b_256( + ::scale::encode(std::tie(descriptor, commitments_hash)).value())); + } + return hash_.value(); + } + + SCALE_TIE_ONLY(descriptor, commitments_hash); + }; + + struct CommittedCandidateReceipt { + SCALE_TIE(2); + + CandidateDescriptor descriptor; /// Candidate descriptor + CandidateCommitments commitments; /// commitments retrieved from validation + /// result and produced by the execution + /// and validation parachain candidate + + CandidateReceipt to_plain(const crypto::Hasher &hasher) const { + CandidateReceipt receipt; + receipt.descriptor = descriptor, + receipt.commitments_hash = + hasher.blake2b_256(scale::encode(commitments).value()); + return receipt; + } + }; + } // namespace kagome::network namespace kagome::parachain::fragment { diff --git a/core/parachain/validator/impl/candidates.hpp b/core/parachain/validator/impl/candidates.hpp index 6add3563b2..3411e34af6 100644 --- a/core/parachain/validator/impl/candidates.hpp +++ b/core/parachain/validator/impl/candidates.hpp @@ -6,6 +6,7 @@ #pragma once #include +#include #include #include #include @@ -14,6 +15,8 @@ #include #include "parachain/validator/collations.hpp" #include "primitives/common.hpp" +#include "utils/map.hpp" +#include "utils/retain_if.hpp" namespace kagome::parachain { @@ -96,6 +99,60 @@ namespace kagome::parachain { }); } + bool has_claims() const { + return !claims.empty(); + } + + template + void on_deactivate_leaves(std::span leaves, + F &&remove_parent_index, + D &&relay_parent_live) { + retain_if(claims, [&](const auto &c) { + if (std::forward(relay_parent_live)(c.second.relay_parent)) { + return true; + } + + if (c.second.parent_hash_and_id) { + const auto &pc = *c.second.parent_hash_and_id; + if (auto it_1 = parent_claims.find(pc.first); + it_1 != parent_claims.end()) { + if (auto it_2 = it_1->second.find(pc.second); + it_2 != it_1->second.end()) { + auto it = std::ranges::find_if(it_2->second, [&](const auto &x) { + return x.first == c.second.relay_parent; + }); + if (it != it_2->second.end()) { + const auto p = std::distance(it_2->second.begin(), it); + auto &sub_claims = it_2->second; + sub_claims[p].second -= 1; + if (sub_claims[p].second == 0) { + auto rem_it = sub_claims.begin(); + std::advance(rem_it, p); + sub_claims.erase(rem_it); + } + } + + if (it_2->second.empty()) { + std::forward(remove_parent_index)(pc.first, pc.second); + it_1->second.erase(it_2); + } + } + + if (it_1->second.empty()) { + parent_claims.erase(it_1); + } + } + } + return false; + }); + + retain_if(unconfirmed_importable_under, [&](const auto &pair) { + const auto &[l, props] = pair; + return (std::ranges::find(leaves, l) != leaves.end()) + && std::forward(relay_parent_live)(props.relay_parent); + }); + } + void extend_hypotheticals( const CandidateHash &candidate_hash, std::vector &v, @@ -210,6 +267,7 @@ namespace kagome::parachain { Hash, std::unordered_map>> by_parent; + log::Logger logger = log::createLogger("Candidates", "parachain"); std::vector frontier_hypotheticals( const std::optional, @@ -441,6 +499,55 @@ namespace kagome::parachain { .reckoning = reckoning, }; } + + void on_deactivate_leaves( + std::span leaves, + const std::function &relay_parent_live) { + auto remove_parent_claims = + [&](const auto &c_hash, const auto &parent_hash, const auto id) { + if (auto it_1 = utils::get_it(by_parent, parent_hash)) { + if (auto it_2 = utils::get_it((*it_1)->second, id)) { + (*it_2)->second.erase(c_hash); + if ((*it_2)->second.empty()) { + (*it_1)->second.erase(*it_2); + } + } + if ((*it_1)->second.empty()) { + by_parent.erase(*it_1); + } + } + }; + + retain_if(candidates, [&](auto &pair) { + auto &[c_hash, state] = pair; + return visit_in_place( + state, + [&](ConfirmedCandidate &c) { + if (!relay_parent_live(c.relay_parent())) { + remove_parent_claims( + c_hash, c.parent_head_data_hash(), c.para_id()); + return false; + } + + for (const auto &leaf_hash : leaves) { + c.importable_under.erase(leaf_hash); + } + return true; + }, + [&](UnconfirmedCandidate &c) { + c.on_deactivate_leaves( + leaves, + [&](const auto &parent_hash, const auto &id) { + return remove_parent_claims(c_hash, parent_hash, id); + }, + relay_parent_live); + return c.has_claims(); + }); + }); + + SL_TRACE( + logger, "Candidates remaining after cleanup: {}", candidates.size()); + } }; } // namespace kagome::parachain diff --git a/core/parachain/validator/impl/parachain_observer_impl.cpp b/core/parachain/validator/impl/parachain_observer_impl.cpp index 4495ec5ec4..1f3e281118 100644 --- a/core/parachain/validator/impl/parachain_observer_impl.cpp +++ b/core/parachain/validator/impl/parachain_observer_impl.cpp @@ -139,7 +139,7 @@ namespace kagome::parachain { primitives::BlockHash relay_parent, std::optional> &&prospective_candidate, network::CollationVersion collator_protocol_version) { - processor_->handleAdvertisement( + processor_->handle_advertisement( relay_parent, peer_id, std::move(prospective_candidate)); } diff --git a/core/parachain/validator/impl/parachain_processor.cpp b/core/parachain/validator/impl/parachain_processor.cpp index 3018aae47e..b8400f9b2f 100644 --- a/core/parachain/validator/impl/parachain_processor.cpp +++ b/core/parachain/validator/impl/parachain_processor.cpp @@ -157,7 +157,8 @@ namespace kagome::parachain { std::shared_ptr prospective_parachains, std::shared_ptr block_tree, LazySPtr slots_util, - std::shared_ptr babe_config_repo) + std::shared_ptr babe_config_repo, + std::shared_ptr sd) : pm_(std::move(pm)), runtime_info_(std::move(runtime_info)), crypto_provider_(std::move(crypto_provider)), @@ -176,15 +177,14 @@ namespace kagome::parachain { app_config_(app_config), sync_state_observable_(std::move(sync_state_observable)), query_audi_{std::move(query_audi)}, - per_session_(RefCache::create()), - peer_use_count_( - std::make_shared()), slots_util_(slots_util), babe_config_repo_(std::move(babe_config_repo)), chain_sub_{std::move(chain_sub_engine)}, worker_pool_handler_{worker_thread_pool.handler(app_state_manager)}, prospective_parachains_{std::move(prospective_parachains)}, - block_tree_{std::move(block_tree)} { + block_tree_{std::move(block_tree)}, + statement_distribution(std::move(sd)), + per_session(RefCache::create()) { BOOST_ASSERT(pm_); BOOST_ASSERT(peer_view_); BOOST_ASSERT(crypto_provider_); @@ -204,6 +204,7 @@ namespace kagome::parachain { BOOST_ASSERT(prospective_parachains_); BOOST_ASSERT(worker_pool_handler_); BOOST_ASSERT(block_tree_); + BOOST_ASSERT(statement_distribution); app_state_manager.takeControl(*this); our_current_state_.implicit_view.emplace( @@ -245,6 +246,7 @@ namespace kagome::parachain { * @return true if the preparation is successful. */ bool ParachainProcessorImpl::prepare() { + statement_distribution->store_parachain_processor(weak_from_this()); // Set the broadcast callback for the bitfield signer bitfield_signer_->setBroadcastCallback( [wptr_self{weak_from_this()}](const primitives::BlockHash &relay_parent, @@ -309,277 +311,9 @@ namespace kagome::parachain { self->onViewUpdated(event); }); - remote_view_sub_ = std::make_shared( - peer_view_->getRemoteViewObservable(), false); - primitives::events::subscribe( - *remote_view_sub_, - network::PeerView::EventType::kViewUpdated, - [wptr{weak_from_this()}](const libp2p::peer::PeerId &peer_id, - const network::View &view) { - TRY_GET_OR_RET(self, wptr.lock()); - self->onUpdatePeerView(peer_id, view); - }); - return true; } - void ParachainProcessorImpl::onUpdatePeerView( - const libp2p::peer::PeerId &peer, const network::View &new_view) { - REINVOKE(*main_pool_handler_, onUpdatePeerView, peer, new_view); - TRY_GET_OR_RET(peer_state, pm_->getPeerState(peer)); - - auto fresh_implicit = peer_state->get().update_view( - new_view, *our_current_state_.implicit_view); - for (const auto &new_relay_parent : fresh_implicit) { - send_peer_messages_for_relay_parent(peer, new_relay_parent); - } - } - - void ParachainProcessorImpl::send_pending_cluster_statements( - const RelayHash &relay_parent, - const libp2p::peer::PeerId &peer_id, - network::CollationVersion version, - ValidatorIndex peer_validator_id, - ParachainProcessorImpl::RelayParentState &relay_parent_state) { - CHECK_OR_RET(relay_parent_state.local_validator); - CHECK_OR_RET(relay_parent_state.local_validator->active); - - const auto pending_statements = - relay_parent_state.local_validator->active->cluster_tracker - .pending_statements_for(peer_validator_id); - std::deque, - network::VersionedValidatorProtocolMessage>> - messages; - for (const auto &[originator, compact] : pending_statements) { - if (!candidates_.is_confirmed(candidateHash(compact))) { - continue; - } - - auto res = - pending_statement_network_message(*relay_parent_state.statement_store, - relay_parent, - peer_id, - version, - originator, - network::vstaging::from(compact)); - - if (res) { - relay_parent_state.local_validator->active->cluster_tracker.note_sent( - peer_validator_id, originator, compact); - messages.emplace_back(*res); - } - } - - auto se = pm_->getStreamEngine(); - for (auto &[peers, msg] : messages) { - if (auto m = if_type(msg)) { - auto message = std::make_shared< - network::WireMessage>( - std::move(m->get())); - for (const auto &p : peers) { - se->send(p, router_->getValidationProtocolVStaging(), message); - } - } else { - BOOST_ASSERT(false); - } - } - } - - void ParachainProcessorImpl::send_pending_grid_messages( - const RelayHash &relay_parent, - const libp2p::peer::PeerId &peer_id, - network::CollationVersion version, - ValidatorIndex peer_validator_id, - const Groups &groups, - ParachainProcessorImpl::RelayParentState &relay_parent_state) { - CHECK_OR_RET(relay_parent_state.local_validator); - - auto pending_manifests = - relay_parent_state.local_validator->grid_tracker.pending_manifests_for( - peer_validator_id); - std::deque, - network::VersionedValidatorProtocolMessage>> - messages; - for (const auto &[candidate_hash, kind] : pending_manifests) { - const auto confirmed_candidate = - candidates_.get_confirmed(candidate_hash); - if (!confirmed_candidate) { - continue; - } - - const auto group_index = confirmed_candidate->get().group_index(); - TRY_GET_OR_RET(group, groups.get(group_index)); - - const auto group_size = group->size(); - auto local_knowledge = - local_knowledge_filter(group_size, - group_index, - candidate_hash, - *relay_parent_state.statement_store); - - switch (kind) { - case grid::ManifestKind::Full: { - const network::vstaging::BackedCandidateManifest manifest{ - .relay_parent = relay_parent, - .candidate_hash = candidate_hash, - .group_index = group_index, - .para_id = confirmed_candidate->get().para_id(), - .parent_head_data_hash = - confirmed_candidate->get().parent_head_data_hash(), - .statement_knowledge = local_knowledge, - }; - - auto &grid = relay_parent_state.local_validator->grid_tracker; - grid.manifest_sent_to( - groups, peer_validator_id, candidate_hash, local_knowledge); - - switch (version) { - case network::CollationVersion::VStaging: { - messages.emplace_back( - std::vector{peer_id}, - network::VersionedValidatorProtocolMessage{ - kagome::network::vstaging::ValidatorProtocolMessage{ - kagome::network::vstaging:: - StatementDistributionMessage{manifest}}}); - } break; - default: { - SL_ERROR(logger_, - "Bug ValidationVersion::V1 should not be used in " - "statement-distribution v2, legacy should have handled " - "this."); - } break; - }; - } break; - case grid::ManifestKind::Acknowledgement: { - auto m = acknowledgement_and_statement_messages( - peer_id, - network::CollationVersion::VStaging, - peer_validator_id, - groups, - relay_parent_state, - relay_parent, - group_index, - candidate_hash, - local_knowledge); - messages.insert(messages.end(), - std::move_iterator(m.begin()), - std::move_iterator(m.end())); - - } break; - } - } - - { - auto &grid_tracker = relay_parent_state.local_validator->grid_tracker; - auto pending_statements = - grid_tracker.all_pending_statements_for(peer_validator_id); - - for (const auto &[originator, compact] : pending_statements) { - auto res = pending_statement_network_message( - *relay_parent_state.statement_store, - relay_parent, - peer_id, - network::CollationVersion::VStaging, - originator, - compact); - - if (res) { - grid_tracker.sent_or_received_direct_statement( - groups, originator, peer_validator_id, compact, false); - - messages.emplace_back(std::move(*res)); - } - } - } - - auto se = pm_->getStreamEngine(); - for (auto &[peers, msg] : messages) { - if (auto m = if_type(msg)) { - auto message = std::make_shared< - network::WireMessage>( - std::move(m->get())); - for (const auto &p : peers) { - se->send(p, router_->getValidationProtocolVStaging(), message); - } - } else { - assert(false); - } - } - } - - std::optional, - network::VersionedValidatorProtocolMessage>> - ParachainProcessorImpl::pending_statement_network_message( - const StatementStore &statement_store, - const RelayHash &relay_parent, - const libp2p::peer::PeerId &peer, - network::CollationVersion version, - ValidatorIndex originator, - const network::vstaging::CompactStatement &compact) { - switch (version) { - case network::CollationVersion::VStaging: { - auto s = statement_store.validator_statement(originator, compact); - if (s) { - return std::make_pair( - std::vector{peer}, - network::VersionedValidatorProtocolMessage{ - network::vstaging::ValidatorProtocolMessage{ - network::vstaging::StatementDistributionMessage{ - network::vstaging:: - StatementDistributionMessageStatement{ - .relay_parent = relay_parent, - .compact = s->get().statement, - }}}}); - } - } break; - default: { - SL_ERROR(logger_, - "Bug ValidationVersion::V1 should not be used in " - "statement-distribution v2, legacy should have handled this"); - } break; - } - return {}; - } - - void ParachainProcessorImpl::send_peer_messages_for_relay_parent( - const libp2p::peer::PeerId &peer_id, const RelayHash &relay_parent) { - BOOST_ASSERT( - main_pool_handler_ - ->isInCurrentThread()); // because of pm_->getPeerState(...) - - TRY_GET_OR_RET(peer_state, pm_->getPeerState(peer_id)); - TRY_GET_OR_RET(parachain_state, tryGetStateByRelayParent(relay_parent)); - - network::CollationVersion version = network::CollationVersion::VStaging; - if (peer_state->get().collation_version) { - version = *peer_state->get().collation_version; - } - - if (auto auth_id = query_audi_->get(peer_id)) { - if (auto it = parachain_state->get().authority_lookup.find(*auth_id); - it != parachain_state->get().authority_lookup.end()) { - ValidatorIndex vi = it->second; - - SL_TRACE(logger_, - "Send pending cluster/grid messages. (peer={}. validator " - "index={}, relay_parent={})", - peer_id, - vi, - relay_parent); - send_pending_cluster_statements( - relay_parent, peer_id, version, vi, parachain_state->get()); - - send_pending_grid_messages( - relay_parent, - peer_id, - version, - vi, - parachain_state->get().per_session_state->value().groups, - parachain_state->get()); - } - } - } - void ParachainProcessorImpl::onViewUpdated(const network::ExView &event) { REINVOKE(*main_pool_handler_, onViewUpdated, event); CHECK_OR_RET(canProcessParachains().has_value()); @@ -603,101 +337,85 @@ namespace kagome::parachain { backing_store_->onActivateLeaf(relay_parent); /// init `backing` subsystem - create_backing_task(relay_parent, event.new_head, event.lost); + auto pruned = create_backing_task(relay_parent, event.new_head, event.lost); - /// update our `view` on remote nodes SL_TRACE(logger_, "Update my view.(new head={}, finalized={}, leaves={})", relay_parent, event.view.finalized_number_, event.view.heads_.size()); broadcastView(event.view); - broadcastViewToGroup(relay_parent, event.view); - /// update `statements_distribution` subsystem - { - auto new_relay_parents = - our_current_state_.implicit_view->all_allowed_relay_parents(); - std::vector>> - update_peers; - pm_->enumeratePeerState([&](const libp2p::peer::PeerId &peer, - network::PeerState &peer_state) { - std::vector fresh = - peer_state.reconcile_active_leaf(relay_parent, new_relay_parents); - if (!fresh.empty()) { - update_peers.emplace_back(peer, fresh); - } - return true; - }); - for (const auto &[peer, fresh] : update_peers) { - for (const auto &fresh_relay_parent : fresh) { - send_peer_messages_for_relay_parent(peer, fresh_relay_parent); - } - } - } - new_leaf_fragment_chain_updates(relay_parent); - - // need to lock removing session infoes - for (const auto &lost : event.lost) { - our_current_state_.active_leaves.erase(lost); + handle_active_leaves_update_for_validator(event, std::move(pruned)); + } - { /// remove cancelations - auto &container = our_current_state_.collation_requests_cancel_handles; - for (auto pc = container.begin(); pc != container.end();) { - if (pc->relay_parent != lost) { - ++pc; - } else { - pc = container.erase(pc); - } - } + void ParachainProcessorImpl::handle_active_leaves_update_for_validator( + const network::ExView &event, std::vector pruned_h) { + const auto current_leaves = our_current_state_.validator_side.active_leaves; + std::unordered_map removed; + for (const auto &[h, m] : current_leaves) { + if (!event.view.contains(h)) { + removed.emplace(h, m); } - { /// remove fetched candidates - auto &container = our_current_state_.validator_side.fetched_candidates; - for (auto pc = container.begin(); pc != container.end();) { - if (pc->first.relay_parent != lost) { - ++pc; - } else { - pc = container.erase(pc); - } - } + } + std::vector added; + for (const auto &h : event.view.heads_) { + if (!current_leaves.contains(h)) { + added.emplace_back(h); } + } - av_store_->remove(lost); + for (const auto &leaf : added) { + const auto mode = + prospective_parachains_->prospectiveParachainsMode(leaf); + our_current_state_.validator_side.active_leaves[leaf] = mode; } - our_current_state_.active_leaves[relay_parent] = - prospective_parachains_->prospectiveParachainsMode(relay_parent); + for (const auto &[removed, mode] : removed) { + our_current_state_.validator_side.active_leaves.erase(removed); + const std::vector pruned = + mode ? std::move(pruned_h) : std::vector{removed}; - auto remove_if = [](bool eq, auto &it, auto &cont) { - if (eq) { - it = cont.erase(it); - } else { - ++it; - } - }; + for (const auto &removed : pruned) { + our_current_state_.state_by_relay_parent.erase(removed); - // para - for (auto it = our_current_state_.blocked_advertisements.begin(); - it != our_current_state_.blocked_advertisements.end();) { - // hash - for (auto it_2 = it->second.begin(); it_2 != it->second.end();) { - // adv - for (auto it_3 = it_2->second.begin(); it_3 != it_2->second.end();) { - remove_if(!tryGetStateByRelayParent(it_3->candidate_relay_parent), - it_3, - it_2->second); + { /// remove cancelations + auto &container = + our_current_state_.collation_requests_cancel_handles; + for (auto pc = container.begin(); pc != container.end();) { + if (pc->relay_parent != removed) { + ++pc; + } else { + pc = container.erase(pc); + } + } + } + { /// remove fetched candidates + auto &container = + our_current_state_.validator_side.fetched_candidates; + for (auto pc = container.begin(); pc != container.end();) { + if (pc->first.relay_parent != removed) { + ++pc; + } else { + pc = container.erase(pc); + } + } } - remove_if(it_2->second.empty(), it_2, it->second); } - remove_if( - it->second.empty(), it, our_current_state_.blocked_advertisements); } - auto maybe_unblocked = std::move(our_current_state_.blocked_advertisements); - requestUnblockedCollations(std::move(maybe_unblocked)); + retain_if(our_current_state_.validator_side.blocked_from_seconding, + [&](auto &pair) { + auto &collations = pair.second; + retain_if(collations, [&](const auto &collation) { + return our_current_state_.state_by_relay_parent.contains( + collation.candidate_receipt.descriptor.relay_parent); + }); + return !collations.empty(); + }); prune_old_advertisements(*our_current_state_.implicit_view, - our_current_state_.active_leaves, + our_current_state_.validator_side.active_leaves, our_current_state_.state_by_relay_parent); printStoragesLoad(); } @@ -715,27 +433,6 @@ namespace kagome::parachain { backing_store_->onDeactivateLeaf(lost.hash); bitfield_store_->remove(lost.hash); } - - for (auto it = our_current_state_.state_by_relay_parent.begin(); - it != our_current_state_.state_by_relay_parent.end();) { - const auto &hash = it->first; - const auto &per_relay_state = it->second; - const auto header = block_tree_->getBlockHeader(hash); - - const bool keep = header.has_value() - && per_relay_state.prospective_parachains_mode - && (header.value().number - + per_relay_state.prospective_parachains_mode - ->allowed_ancestry_len - + 1) - >= event.finalized; - if (keep) { - ++it; - } else { - our_current_state_.implicit_view->deactivate_leaf(hash); - it = our_current_state_.state_by_relay_parent.erase(it); - } - } } void ParachainProcessorImpl::broadcastViewExcept( @@ -797,8 +494,6 @@ namespace kagome::parachain { network::ViewUpdate{.view = view}); pm_->getStreamEngine()->broadcast(router_->getCollationProtocolVStaging(), msg); - pm_->getStreamEngine()->broadcast(router_->getValidationProtocolVStaging(), - msg); } outcome::result> @@ -839,74 +534,6 @@ namespace kagome::parachain { } } - ParachainProcessorImpl::PerSessionState::PerSessionState( - SessionIndex _session, - runtime::SessionInfo _session_info, - Groups &&_groups, - grid::Views &&_grid_view, - std::optional _our_index, - std::shared_ptr peers) - : session{_session}, - session_info{std::move(_session_info)}, - groups{std::move(_groups)}, - grid_view{std::move(_grid_view)}, - our_index{_our_index}, - peers{std::move(peers)} { - if (our_index) { - our_group = groups.byValidatorIndex(*our_index); - } - if (our_group) { - BOOST_ASSERT(*our_group < session_info.validator_groups.size()); - if (grid_view) { - BOOST_ASSERT(*our_group < grid_view->size()); - } - } - updatePeers(true); - } - - ParachainProcessorImpl::PerSessionState::~PerSessionState() { - updatePeers(false); - } - - void ParachainProcessorImpl::PerSessionState::updatePeers(bool add) const { - if (not our_index or not our_group or not this->peers) { - return; - } - auto &peers = *this->peers; - SAFE_UNIQUE(peers) { - auto f = [&](ValidatorIndex i) { - auto &id = session_info.discovery_keys[i]; - auto it = peers.find(id); - if (add) { - if (it == peers.end()) { - it = peers.emplace(id, 0).first; - } - ++it->second; - } else { - if (it == peers.end()) { - throw std::logic_error{"inconsistent PeerUseCount"}; - } - --it->second; - if (it->second == 0) { - peers.erase(it); - } - } - }; - for (auto &i : session_info.validator_groups[*our_group]) { - f(i); - } - if (grid_view) { - auto &view = grid_view->at(*our_group); - for (auto &i : view.sending) { - f(i); - } - for (auto &i : view.receiving) { - f(i); - } - } - }; - } - outcome::result> ParachainProcessorImpl::fetch_claim_queue(const RelayHash &relay_parent) { constexpr uint32_t CLAIM_QUEUE_RUNTIME_REQUIREMENT = 11; @@ -964,9 +591,6 @@ namespace kagome::parachain { parachain_host_->session_index_for_child(relay_parent)); OUTCOME_TRY(session_info, parachain_host_->session_info(relay_parent, session_index)); - OUTCOME_TRY(randomness, getBabeRandomness(relay_parent)); - OUTCOME_TRY(disabled_validators_, - parachain_host_->disabled_validators(relay_parent)); const auto &[validator_groups, group_rotation_info] = groups; if (!validator) { @@ -1014,45 +638,15 @@ namespace kagome::parachain { return Error::NOT_A_VALIDATOR; } - auto per_session_state = per_session_->get_or_insert(session_index, [&] { - grid::Views grid_view = grid::makeViews( - session_info->validator_groups, - grid::shuffle(session_info->discovery_keys.size(), randomness), - *global_v_index); - - return RefCache::RefObj( - session_index, - *session_info, - Groups{session_info->validator_groups, minimum_backing_votes}, - std::move(grid_view), - validator_index, - peer_use_count_); - }); - - std::unordered_set peers_sent; - std::optional our_group; - if (validator_index) { - our_group = - per_session_state->value().groups.byValidatorIndex(*validator_index); - if (our_group) { - /// update peers of our group - const auto &group = session_info->validator_groups[*our_group]; - for (const auto vi : group) { - spawn_and_update_peer(peers_sent, session_info->discovery_keys[vi]); - } - } - } - - /// update peers in grid view - const auto &grid_view = *per_session_state->value().grid_view; - for (const auto &view : grid_view) { - for (const auto vi : view.sending) { - spawn_and_update_peer(peers_sent, session_info->discovery_keys[vi]); - } - for (const auto vi : view.receiving) { - spawn_and_update_peer(peers_sent, session_info->discovery_keys[vi]); - } - } + OUTCOME_TRY(per_session_state, + per_session->get_or_insert( + session_index, + [&]() -> outcome::result< + RefCache::RefObj> { + return outcome::success( + RefCache::RefObj( + session_index, *session_info)); + })); const auto n_cores = cores.size(); std::unordered_map> out_groups; @@ -1103,62 +697,12 @@ namespace kagome::parachain { } } - std::unordered_map - authority_lookup; - for (ValidatorIndex v = 0; - v < per_session_state->value().session_info.discovery_keys.size(); - ++v) { - authority_lookup[per_session_state->value() - .session_info.discovery_keys[v]] = v; - } - - std::optional statement_store; - std::optional local_validator; - if (mode) { - statement_store.emplace(per_session_state->value().groups); - auto maybe_claim_queue = - [&]() -> std::optional { - auto r = fetch_claim_queue(relay_parent); - if (r.has_value()) { - return r.value(); - } - return std::nullopt; - }(); - - const auto seconding_limit = mode->max_candidate_depth + 1; - local_validator = [&]() -> std::optional { - if (!global_v_index) { - return std::nullopt; - } - if (validator_index) { - return find_active_validator_state(*validator_index, - per_session_state->value().groups, - cores, - group_rotation_info, - maybe_claim_queue, - seconding_limit, - mode->max_candidate_depth); - } - return LocalValidatorState{}; - }(); - } - - std::unordered_set disabled_validators{ - disabled_validators_.begin(), disabled_validators_.end()}; - if (!disabled_validators.empty()) { - SL_TRACE(logger_, - "Disabled validators detected. (relay parent={})", - relay_parent); - } - SL_VERBOSE(logger_, "Inited new backing task v3.(assigned_para={}, " - "assigned_core={}, our index={}, our_group={}, relay " - "parent={})", + "assigned_core={}, our index={}, relay parent={})", assigned_para, assigned_core, global_v_index, - our_group, relay_parent); return RelayParentState{ @@ -1166,9 +710,6 @@ namespace kagome::parachain { .assigned_core = assigned_core, .assigned_para = assigned_para, .validator_to_group = std::move(validator_to_group), - .per_session_state = per_session_state, - .our_index = validator_index, - .our_group = our_group, .collations = {}, .table_context = TableContext{ @@ -1176,82 +717,20 @@ namespace kagome::parachain { .groups = std::move(out_groups), .validators = std::move(validators), }, - .statement_store = std::move(statement_store), .availability_cores = cores, .group_rotation_info = group_rotation_info, .minimum_backing_votes = minimum_backing_votes, - .authority_lookup = std::move(authority_lookup), - .local_validator = local_validator, .awaiting_validation = {}, .issued_statements = {}, .peers_advertised = {}, .fallbacks = {}, .backed_hashes = {}, - .disabled_validators = std::move(disabled_validators), .inject_core_index = inject_core_index, + .per_session_state = per_session_state, }; } - std::optional - ParachainProcessorImpl::find_active_validator_state( - ValidatorIndex validator_index, - const Groups &groups, - const std::vector &availability_cores, - const runtime::GroupDescriptor &group_rotation_info, - const std::optional &maybe_claim_queue, - size_t seconding_limit, - size_t max_candidate_depth) { - if (groups.all_empty()) { - return std::nullopt; - } - - const auto our_group = groups.byValidatorIndex(validator_index); - if (!our_group) { - return std::nullopt; - } - - const auto core_index = - group_rotation_info.coreForGroup(*our_group, availability_cores.size()); - std::optional para_assigned_to_core; - if (maybe_claim_queue) { - para_assigned_to_core = maybe_claim_queue->get_claim_for(core_index, 0); - } else { - if (core_index < availability_cores.size()) { - const auto &core_state = availability_cores[core_index]; - visit_in_place( - core_state, - [&](const runtime::ScheduledCore &scheduled) { - para_assigned_to_core = scheduled.para_id; - }, - [&](const runtime::OccupiedCore &occupied) { - if (max_candidate_depth >= 1 && occupied.next_up_on_available) { - para_assigned_to_core = occupied.next_up_on_available->para_id; - } - }, - [](const auto &) {}); - } - } - - const auto group_validators = groups.get(*our_group); - if (!group_validators) { - return std::nullopt; - } - - return LocalValidatorState{ - .grid_tracker = {}, - .active = - ActiveValidatorState{ - .index = validator_index, - .group = *our_group, - .assignment = para_assigned_to_core, - .cluster_tracker = ClusterTracker( - {group_validators->begin(), group_validators->end()}, - seconding_limit), - }, - }; - } - - void ParachainProcessorImpl::create_backing_task( + std::vector ParachainProcessorImpl::create_backing_task( const primitives::BlockHash &relay_parent, const network::HashedBlockHeader &block_header, const std::vector &lost) { @@ -1274,9 +753,10 @@ namespace kagome::parachain { res = std::nullopt; } - for (const auto &deactivated : lost) { - our_current_state_.per_leaf.erase(deactivated); - our_current_state_.implicit_view->deactivate_leaf(deactivated); + std::vector pruned; + for (const auto &l : lost) { + our_current_state_.per_leaf.erase(l); + pruned = our_current_state_.implicit_view->deactivate_leaf(l); } std::vector< @@ -1318,7 +798,7 @@ namespace kagome::parachain { ProspectiveParachainsModeOpt leaf_mode; if (!res) { if (our_current_state_.per_leaf.contains(relay_parent)) { - return; + return pruned; } our_current_state_.per_leaf.insert_or_assign(relay_parent, @@ -1350,7 +830,7 @@ namespace kagome::parachain { relay_parent, res->error()); - return; + return pruned; } for (const auto &maybe_new : fresh_relay_parents) { @@ -1377,6 +857,52 @@ namespace kagome::parachain { rps_result.error()); } } + + return pruned; + } + + void ParachainProcessorImpl::second_unblocked_collations( + ParachainId para_id, + const HeadData &head_data, + const Hash &head_data_hash) { + auto unblocked_collations_it = + our_current_state_.validator_side.blocked_from_seconding.find( + BlockedCollationId(para_id, head_data_hash)); + + if (unblocked_collations_it + != our_current_state_.validator_side.blocked_from_seconding.end()) { + auto &unblocked_collations = unblocked_collations_it->second; + + if (!unblocked_collations.empty()) { + SL_TRACE(logger_, + "Candidate outputting head data with hash {} unblocked {} " + "collations for seconding.", + head_data_hash, + unblocked_collations.size()); + } + + for (auto &unblocked_collation : unblocked_collations) { + unblocked_collation.maybe_parent_head_data = head_data; + const auto peer_id = + unblocked_collation.collation_event.pending_collation.peer_id; + const auto relay_parent = + unblocked_collation.candidate_receipt.descriptor.relay_parent; + + if (auto res = kick_off_seconding(std::move(unblocked_collation)); + res.has_error()) { + SL_WARN(logger_, + "Seconding aborted due to an error. (relay_parent={}, " + "para_id={}, peer_id={}, error={})", + relay_parent, + para_id, + peer_id, + res.error()); + } + } + + our_current_state_.validator_side.blocked_from_seconding.erase( + unblocked_collations_it); + } } void ParachainProcessorImpl::handle_collation_fetch_response( @@ -1466,6 +992,12 @@ namespace kagome::parachain { pending_collation_copy.peer_id, res.error()); + const auto maybe_candidate_hash = + utils::map(pending_collation_copy.prospective_candidate, + [](const auto &v) { return v.candidate_hash; }); + dequeue_next_collation_and_fetch(pending_collation_copy.relay_parent, + {collator_id, maybe_candidate_hash}); + } else if (res.value() == false) { const auto maybe_candidate_hash = utils::map(pending_collation_copy.prospective_candidate, [](const auto &v) { return v.candidate_hash; }); @@ -1481,7 +1013,9 @@ namespace kagome::parachain { 32, crypto::Blake2b_StreamHasher<32>> &persisted_validation_data, - std::optional> maybe_parent_head_and_hash) { + std::optional, + std::reference_wrapper>> + maybe_parent_head_and_hash) { if (persisted_validation_data.getHash() != fetched.descriptor.persisted_data_hash) { return Error::PERSISTED_VALIDATION_DATA_MISMATCH; @@ -1494,8 +1028,8 @@ namespace kagome::parachain { } if (maybe_parent_head_and_hash - && hasher_->blake2b_256(maybe_parent_head_and_hash->first) - != maybe_parent_head_and_hash->second) { + && hasher_->blake2b_256(maybe_parent_head_and_hash->first.get()) + != maybe_parent_head_and_hash->second.get()) { return Error::PARENT_HEAD_DATA_MISMATCH; } @@ -1623,195 +1157,45 @@ namespace kagome::parachain { bitfield_store_->putBitfield(bd->relay_parent, bd->data); } - ParachainProcessorImpl::ManifestImportSuccessOpt - ParachainProcessorImpl::handle_incoming_manifest_common( - const libp2p::peer::PeerId &peer_id, - const CandidateHash &candidate_hash, + void ParachainProcessorImpl::send_to_validators_group( const RelayHash &relay_parent, - ManifestSummary manifest_summary, - ParachainId para_id, - grid::ManifestKind manifest_kind) { - auto peer_state = pm_->getPeerState(peer_id); - if (!peer_state) { - SL_WARN(logger_, "No peer state. (peer_id={})", peer_id); - return {}; - } + const std::deque &messages) { + BOOST_ASSERT(main_pool_handler_->isInCurrentThread()); - auto relay_parent_state = tryGetStateByRelayParent(relay_parent); - if (!relay_parent_state) { - return {}; + auto se = pm_->getStreamEngine(); + std::unordered_set group_set; + if (auto r = runtime_info_->get_session_info(relay_parent)) { + auto &[session, info] = r.value(); + if (info.our_group) { + for (auto &i : session.validator_groups[*info.our_group]) { + if (auto peer = query_audi_->get(session.discovery_keys[i])) { + group_set.emplace(peer->id); + } + } + } } - if (!relay_parent_state->get().local_validator) { - return {}; + std::deque group, any; + for (const auto &p : group_set) { + group.emplace_back(p); } - auto expected_group = - group_for_para(relay_parent_state->get().availability_cores, - relay_parent_state->get().group_rotation_info, - para_id); - - if (!expected_group - || *expected_group != manifest_summary.claimed_group_index) { - return {}; - } + auto protocol = [&]() -> std::shared_ptr { + return router_->getValidationProtocolVStaging(); + }(); - if (!relay_parent_state->get().per_session_state->value().grid_view) { - return {}; - } - - const auto &grid_topology = - *relay_parent_state->get().per_session_state->value().grid_view; - if (manifest_summary.claimed_group_index >= grid_topology.size()) { - return {}; - } - - auto sender_index = [&]() -> std::optional { - const auto &sub = grid_topology[manifest_summary.claimed_group_index]; - const auto &iter = (manifest_kind == grid::ManifestKind::Full) - ? sub.receiving - : sub.sending; - if (!iter.empty()) { - return *iter.begin(); - } - return {}; - }(); - - if (!sender_index) { - return {}; - } - - auto group_index = manifest_summary.claimed_group_index; - auto claimed_parent_hash = manifest_summary.claimed_parent_hash; - - auto group = [&]() -> std::span { - if (auto g = - relay_parent_state->get().per_session_state->value().groups.get( - group_index)) { - return *g; - } - return {}; - }(); - - auto disabled_mask = relay_parent_state->get().disabled_bitmask(group); - manifest_summary.statement_knowledge.mask_seconded(disabled_mask); - manifest_summary.statement_knowledge.mask_valid(disabled_mask); - - BOOST_ASSERT(relay_parent_state->get().prospective_parachains_mode); - const auto seconding_limit = - relay_parent_state->get() - .prospective_parachains_mode->max_candidate_depth - + 1; - - auto &local_validator = *relay_parent_state->get().local_validator; - - SL_TRACE( - logger_, - "Import manifest. (peer_id={}, relay_parent={}, candidate_hash={})", - peer_id, - relay_parent, - candidate_hash); - auto acknowledge_res = local_validator.grid_tracker.import_manifest( - grid_topology, - relay_parent_state->get().per_session_state->value().groups, - candidate_hash, - seconding_limit, - manifest_summary, - manifest_kind, - *sender_index); - - if (acknowledge_res.has_error()) { - SL_WARN(logger_, - "Import manifest failed. (peer_id={}, relay_parent={}, " - "candidate_hash={}, error={})", - peer_id, - relay_parent, - candidate_hash, - acknowledge_res.error()); - return {}; - } - - const auto acknowledge = acknowledge_res.value(); - if (!candidates_.insert_unconfirmed(peer_id, - candidate_hash, - relay_parent, - group_index, - {{claimed_parent_hash, para_id}})) { - SL_TRACE(logger_, - "Insert unconfirmed candidate failed. (candidate hash={}, relay " - "parent={}, para id={}, claimed parent={})", - candidate_hash, - relay_parent, - para_id, - manifest_summary.claimed_parent_hash); - return {}; - } - - if (acknowledge) { - SL_TRACE(logger_, - "immediate ack, known candidate. (candidate hash={}, from={}, " - "local_validator={})", - candidate_hash, - *sender_index, - *relay_parent_state->get().our_index); - } - - return ManifestImportSuccess{ - .acknowledge = acknowledge, - .sender_index = *sender_index, - }; - } - - network::vstaging::StatementFilter - ParachainProcessorImpl::local_knowledge_filter( - size_t group_size, - GroupIndex group_index, - const CandidateHash &candidate_hash, - const StatementStore &statement_store) { - network::vstaging::StatementFilter f{group_size}; - statement_store.fill_statement_filter(group_index, candidate_hash, f); - return f; - } - - void ParachainProcessorImpl::send_to_validators_group( - const RelayHash &relay_parent, - const std::deque &messages) { - BOOST_ASSERT(main_pool_handler_->isInCurrentThread()); - - auto se = pm_->getStreamEngine(); - std::unordered_set group_set; - if (auto r = runtime_info_->get_session_info(relay_parent)) { - auto &[session, info] = r.value(); - if (info.our_group) { - for (auto &i : session.validator_groups[*info.our_group]) { - if (auto peer = query_audi_->get(session.discovery_keys[i])) { - group_set.emplace(peer->id); - } - } - } - } - - std::deque group, any; - for (const auto &p : group_set) { - group.emplace_back(p); - } - - auto protocol = [&]() -> std::shared_ptr { - return router_->getValidationProtocolVStaging(); - }(); - - se->forEachPeer(protocol, [&](const network::PeerId &peer) { - if (not group_set.contains(peer)) { - any.emplace_back(peer); - } - }); - auto lucky = kMinGossipPeers - std::min(group.size(), kMinGossipPeers); - if (lucky != 0) { - std::shuffle(any.begin(), any.end(), random_); - // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) - any.erase(any.begin() + std::min(any.size(), lucky), any.end()); - } else { - any.clear(); + se->forEachPeer(protocol, [&](const network::PeerId &peer) { + if (not group_set.contains(peer)) { + any.emplace_back(peer); + } + }); + auto lucky = kMinGossipPeers - std::min(group.size(), kMinGossipPeers); + if (lucky != 0) { + std::shuffle(any.begin(), any.end(), random_); + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) + any.erase(any.begin() + std::min(any.size(), lucky), any.end()); + } else { + any.clear(); } auto make_send = [&]( @@ -1837,1167 +1221,46 @@ namespace kagome::parachain { } for (auto &peer : any) { - SL_TRACE(logger_, "Send to peer from any. (peer={})", peer); - se->send(peer, protocol, message); - } - }; - - for (const network::VersionedValidatorProtocolMessage &msg : messages) { - visit_in_place( - msg, - [&](const kagome::network::vstaging::ValidatorProtocolMessage &m) { - make_send(m, router_->getValidationProtocolVStaging()); - }, - [&](const kagome::network::ValidatorProtocolMessage &m) { - make_send(m, router_->getValidationProtocol()); - }); - } - } - - std::deque, - network::VersionedValidatorProtocolMessage>> - ParachainProcessorImpl::acknowledgement_and_statement_messages( - const libp2p::peer::PeerId &peer, - network::CollationVersion version, - ValidatorIndex validator_index, - const Groups &groups, - ParachainProcessorImpl::RelayParentState &relay_parent_state, - const RelayHash &relay_parent, - GroupIndex group_index, - const CandidateHash &candidate_hash, - const network::vstaging::StatementFilter &local_knowledge) { - if (!relay_parent_state.local_validator) { - return {}; - } - - auto &local_validator = *relay_parent_state.local_validator; - std::deque, - network::VersionedValidatorProtocolMessage>> - messages; - - switch (version) { - case network::CollationVersion::VStaging: { - messages.emplace_back( - std::vector{peer}, - network::VersionedValidatorProtocolMessage{ - network::vstaging::ValidatorProtocolMessage{ - network::vstaging::StatementDistributionMessage{ - network::vstaging::BackedCandidateAcknowledgement{ - .candidate_hash = candidate_hash, - .statement_knowledge = local_knowledge, - }}}}); - } break; - default: { - SL_ERROR(logger_, - "Bug ValidationVersion::V1 should not be used in " - "statement-distribution v2, legacy should have handled this"); - return {}; - } break; - }; - - local_validator.grid_tracker.manifest_sent_to( - groups, validator_index, candidate_hash, local_knowledge); - - auto statement_messages = post_acknowledgement_statement_messages( - validator_index, - relay_parent, - local_validator.grid_tracker, - *relay_parent_state.statement_store, - groups, - group_index, - candidate_hash, - peer, - version); - - for (auto &&m : statement_messages) { - messages.emplace_back(std::vector{peer}, - std::move(m)); - } - return messages; - } - - std::deque - ParachainProcessorImpl::post_acknowledgement_statement_messages( - ValidatorIndex recipient, - const RelayHash &relay_parent, - grid::GridTracker &grid_tracker, - const StatementStore &statement_store, - const Groups &groups, - GroupIndex group_index, - const CandidateHash &candidate_hash, - const libp2p::peer::PeerId &peer, - network::CollationVersion version) { - auto sending_filter = - grid_tracker.pending_statements_for(recipient, candidate_hash); - if (!sending_filter) { - return {}; - } - - std::deque messages; - auto group = groups.get(group_index); - if (!group) { - return messages; - } - - statement_store.groupStatements( - *group, - candidate_hash, - *sending_filter, - [&](const IndexedAndSigned - &statement) { - grid_tracker.sent_or_received_direct_statement(groups, - statement.payload.ix, - recipient, - getPayload(statement), - false); - - switch (version) { - case network::CollationVersion::VStaging: { - messages.emplace_back(network::vstaging::ValidatorProtocolMessage{ - network::vstaging::StatementDistributionMessage{ - network::vstaging::StatementDistributionMessageStatement{ - .relay_parent = relay_parent, - .compact = statement, - }}}); - } break; - default: { - SL_ERROR( - logger_, - "Bug ValidationVersion::V1 should not be used in " - "statement-distribution v2, legacy should have handled this"); - } break; - } - }); - return messages; - } - - outcome::result ParachainProcessorImpl::handle_grid_statement( - const RelayHash &relay_parent, - ParachainProcessorImpl::RelayParentState &per_relay_parent, - grid::GridTracker &grid_tracker, - const IndexedAndSigned &statement, - ValidatorIndex grid_sender_index) { - /// TODO(iceseer): do Ensure the statement is correctly signed. Signature - /// check. - grid_tracker.sent_or_received_direct_statement( - per_relay_parent.per_session_state->value().groups, - statement.payload.ix, - grid_sender_index, - getPayload(statement), - true); - return outcome::success(); - } - - void ParachainProcessorImpl::handle_incoming_acknowledgement( - const libp2p::peer::PeerId &peer_id, - const network::vstaging::BackedCandidateAcknowledgement - &acknowledgement) { - SL_TRACE(logger_, - "`BackedCandidateAcknowledgement`. (candidate_hash={})", - acknowledgement.candidate_hash); - const auto &candidate_hash = acknowledgement.candidate_hash; - SL_TRACE(logger_, - "Received incoming acknowledgement. (peer={}, candidate hash={})", - peer_id, - candidate_hash); - - TRY_GET_OR_RET(c, candidates_.get_confirmed(candidate_hash)); - const RelayHash &relay_parent = c->get().relay_parent(); - const Hash &parent_head_data_hash = c->get().parent_head_data_hash(); - GroupIndex group_index = c->get().group_index(); - ParachainId para_id = c->get().para_id(); - - TRY_GET_OR_RET(opt_parachain_state, tryGetStateByRelayParent(relay_parent)); - auto &relay_parent_state = opt_parachain_state->get(); - BOOST_ASSERT(relay_parent_state.statement_store); - - SL_TRACE(logger_, - "Handling incoming acknowledgement. (relay_parent={})", - relay_parent); - ManifestImportSuccessOpt x = handle_incoming_manifest_common( - peer_id, - candidate_hash, - relay_parent, - ManifestSummary{ - .claimed_parent_hash = parent_head_data_hash, - .claimed_group_index = group_index, - .statement_knowledge = acknowledgement.statement_knowledge, - }, - para_id, - grid::ManifestKind::Acknowledgement); - CHECK_OR_RET(x); - - SL_TRACE( - logger_, "Check local validator. (relay_parent = {})", relay_parent); - CHECK_OR_RET(relay_parent_state.local_validator); - - const auto sender_index = x->sender_index; - auto &local_validator = *relay_parent_state.local_validator; - - SL_TRACE(logger_, "Post ack. (relay_parent = {})", relay_parent); - auto messages = post_acknowledgement_statement_messages( - sender_index, - relay_parent, - local_validator.grid_tracker, - *relay_parent_state.statement_store, - relay_parent_state.per_session_state->value().groups, - group_index, - candidate_hash, - peer_id, - network::CollationVersion::VStaging); - - auto se = pm_->getStreamEngine(); - SL_TRACE(logger_, "Sending messages. (relay_parent = {})", relay_parent); - for (auto &msg : messages) { - if (auto m = if_type(msg)) { - auto message = std::make_shared< - network::WireMessage>( - std::move(m->get())); - se->send(peer_id, router_->getValidationProtocolVStaging(), message); - } else { - assert(false); - } - } - } - - // Handles BackedCandidateManifest message - // It performs various checks and operations, and if everything is - // successful, it sends acknowledgement and statement messages to the - // validators group or sends a request to fetch the attested candidate. - void ParachainProcessorImpl::handle_incoming_manifest( - const libp2p::peer::PeerId &peer_id, - const network::vstaging::BackedCandidateManifest &manifest) { - SL_TRACE(logger_, - "`BackedCandidateManifest`. (relay_parent={}, " - "candidate_hash={}, para_id={}, parent_head_data_hash={})", - manifest.relay_parent, - manifest.candidate_hash, - manifest.para_id, - manifest.parent_head_data_hash); - - TRY_GET_OR_RET(relay_parent_state, - tryGetStateByRelayParent(manifest.relay_parent)); - CHECK_OR_RET(relay_parent_state->get().statement_store); - - SL_TRACE(logger_, - "Handling incoming manifest common. (relay_parent={}, " - "candidate_hash={})", - manifest.relay_parent, - manifest.candidate_hash); - ManifestImportSuccessOpt x = handle_incoming_manifest_common( - peer_id, - manifest.candidate_hash, - manifest.relay_parent, - ManifestSummary{ - .claimed_parent_hash = manifest.parent_head_data_hash, - .claimed_group_index = manifest.group_index, - .statement_knowledge = manifest.statement_knowledge, - }, - manifest.para_id, - grid::ManifestKind::Full); - CHECK_OR_RET(x); - - const auto sender_index = x->sender_index; - if (x->acknowledge) { - SL_TRACE(logger_, - "Known candidate - acknowledging manifest. (candidate hash={})", - manifest.candidate_hash); - - SL_TRACE(logger_, - "Get groups. (relay_parent={}, candidate_hash={})", - manifest.relay_parent, - manifest.candidate_hash); - auto group = - relay_parent_state->get().per_session_state->value().groups.get( - manifest.group_index); - if (!group) { - return; - } - - network::vstaging::StatementFilter local_knowledge = - local_knowledge_filter(group->size(), - manifest.group_index, - manifest.candidate_hash, - *relay_parent_state->get().statement_store); - SL_TRACE(logger_, - "Get ack and statement messages. (relay_parent={}, " - "candidate_hash={})", - manifest.relay_parent, - manifest.candidate_hash); - auto messages = acknowledgement_and_statement_messages( - peer_id, - network::CollationVersion::VStaging, - sender_index, - relay_parent_state->get().per_session_state->value().groups, - relay_parent_state->get(), - manifest.relay_parent, - manifest.group_index, - manifest.candidate_hash, - local_knowledge); - - SL_TRACE(logger_, - "Send messages. (relay_parent={}, candidate_hash={})", - manifest.relay_parent, - manifest.candidate_hash); - auto se = pm_->getStreamEngine(); - for (auto &[peers, msg] : messages) { - if (auto m = - if_type(msg)) { - auto message = std::make_shared>( - std::move(m->get())); - for (const auto &p : peers) { - se->send(p, router_->getValidationProtocolVStaging(), message); - } - } else { - assert(false); - } - } - } else if (!candidates_.is_confirmed(manifest.candidate_hash)) { - SL_TRACE( - logger_, - "Request attested candidate. (relay_parent={}, candidate_hash={})", - manifest.relay_parent, - manifest.candidate_hash); - request_attested_candidate(peer_id, - relay_parent_state->get(), - manifest.relay_parent, - manifest.candidate_hash, - manifest.group_index); - } - } - - outcome::result< - std::reference_wrapper> - ParachainProcessorImpl::check_statement_signature( - SessionIndex session_index, - const std::vector &validators, - const RelayHash &relay_parent, - const network::vstaging::SignedCompactStatement &statement) { - OUTCOME_TRY(signing_context, - SigningContext::make(parachain_host_, relay_parent)); - OUTCOME_TRY(verified, - crypto_provider_->verify( - statement.signature, - signing_context.signable(*hasher_, getPayload(statement)), - validators[statement.payload.ix])); - - if (!verified) { - return Error::INCORRECT_SIGNATURE; - } - return std::cref(statement); - } - - outcome::result> - ParachainProcessorImpl::handle_cluster_statement( - const RelayHash &relay_parent, - ClusterTracker &cluster_tracker, - SessionIndex session, - const runtime::SessionInfo &session_info, - const network::vstaging::SignedCompactStatement &statement, - ValidatorIndex cluster_sender_index) { - const auto accept = cluster_tracker.can_receive( - cluster_sender_index, - statement.payload.ix, - network::vstaging::from(getPayload(statement))); - if (accept != outcome::success(Accept::Ok) - && accept != outcome::success(Accept::WithPrejudice)) { - SL_ERROR(logger_, "Reject outgoing error."); - return Error::CLUSTER_TRACKER_ERROR; - } - OUTCOME_TRY(check_statement_signature( - session, session_info.validators, relay_parent, statement)); - - cluster_tracker.note_received( - cluster_sender_index, - statement.payload.ix, - network::vstaging::from(getPayload(statement))); - - const auto should_import = (outcome::success(Accept::Ok) == accept); - if (should_import) { - return statement; - } - return std::nullopt; - } - - void ParachainProcessorImpl::handle_incoming_statement( - const libp2p::peer::PeerId &peer_id, - const network::vstaging::StatementDistributionMessageStatement &stm) { - SL_TRACE(logger_, - "`StatementDistributionMessageStatement`. (relay_parent={}, " - "candidate_hash={})", - stm.relay_parent, - candidateHash(getPayload(stm.compact))); - auto parachain_state = tryGetStateByRelayParent(stm.relay_parent); - if (!parachain_state) { - SL_TRACE(logger_, - "After request pov no parachain state on relay_parent. (relay " - "parent={})", - stm.relay_parent); - return; - } - - const auto &session_info = - parachain_state->get().per_session_state->value().session_info; - if (parachain_state->get().is_disabled(stm.compact.payload.ix)) { - SL_TRACE( - logger_, - "Ignoring a statement from disabled validator. (relay parent={}, " - "validator={})", - stm.relay_parent, - stm.compact.payload.ix); - return; - } - - CHECK_OR_RET(parachain_state->get().local_validator); - auto &local_validator = *parachain_state->get().local_validator; - auto originator_group = - parachain_state->get() - .per_session_state->value() - .groups.byValidatorIndex(stm.compact.payload.ix); - if (!originator_group) { - SL_TRACE(logger_, - "No correct validator index in statement. (relay parent={}, " - "validator={})", - stm.relay_parent, - stm.compact.payload.ix); - return; - } - - auto &active = local_validator.active; - auto cluster_sender_index = [&]() -> std::optional { - std::span allowed_senders; - if (active) { - allowed_senders = active->cluster_tracker.senders_for_originator( - stm.compact.payload.ix); - } - - if (auto peer = query_audi_->get(peer_id)) { - for (const auto i : allowed_senders) { - if (i < session_info.discovery_keys.size() - && *peer == session_info.discovery_keys[i]) { - return i; - } - } - } - return std::nullopt; - }(); - - if (active && cluster_sender_index) { - if (handle_cluster_statement( - stm.relay_parent, - active->cluster_tracker, - parachain_state->get().per_session_state->value().session, - parachain_state->get().per_session_state->value().session_info, - stm.compact, - *cluster_sender_index) - .has_error()) { - return; - } - } else { - std::optional> grid_sender_index; - for (const auto &[i, validator_knows_statement] : - local_validator.grid_tracker.direct_statement_providers( - parachain_state->get().per_session_state->value().groups, - stm.compact.payload.ix, - getPayload(stm.compact))) { - if (i >= session_info.discovery_keys.size()) { - continue; - } - - /// TODO(iceseer): do check is authority - /// const auto &ad = opt_session_info->discovery_keys[i]; - grid_sender_index.emplace(i, validator_knows_statement); - break; - } - - CHECK_OR_RET(grid_sender_index); - const auto &[gsi, validator_knows_statement] = *grid_sender_index; - - CHECK_OR_RET(!validator_knows_statement); - if (handle_grid_statement(stm.relay_parent, - parachain_state->get(), - local_validator.grid_tracker, - stm.compact, - gsi) - .has_error()) { - return; - } - } - - const auto &statement = getPayload(stm.compact); - const auto originator_index = stm.compact.payload.ix; - const auto &candidate_hash = candidateHash(getPayload(stm.compact)); - const bool res = candidates_.insert_unconfirmed(peer_id, - candidate_hash, - stm.relay_parent, - *originator_group, - std::nullopt); - CHECK_OR_RET(res); - const auto confirmed = candidates_.get_confirmed(candidate_hash); - const auto is_confirmed = candidates_.is_confirmed(candidate_hash); - const auto &group = session_info.validator_groups[*originator_group]; - - if (!is_confirmed) { - request_attested_candidate(peer_id, - parachain_state->get(), - stm.relay_parent, - candidate_hash, - *originator_group); - } - - /// TODO(iceseer): do https://github.com/qdrvm/kagome/issues/1888 - /// check statement signature - - const auto was_fresh_opt = parachain_state->get().statement_store->insert( - parachain_state->get().per_session_state->value().groups, - stm.compact, - StatementOrigin::Remote); - if (!was_fresh_opt) { - SL_WARN(logger_, - "Accepted message from unknown validator. (relay parent={}, " - "validator={})", - stm.relay_parent, - stm.compact.payload.ix); - return; - } - - if (!*was_fresh_opt) { - SL_TRACE(logger_, - "Statement was not fresh. (relay parent={}, validator={})", - stm.relay_parent, - stm.compact.payload.ix); - return; - } - - const auto is_importable = candidates_.is_importable(candidate_hash); - if (parachain_state->get().per_session_state->value().grid_view) { - local_validator.grid_tracker.learned_fresh_statement( - parachain_state->get().per_session_state->value().groups, - *parachain_state->get().per_session_state->value().grid_view, - originator_index, - statement); - } - - if (is_importable && confirmed) { - send_backing_fresh_statements(confirmed->get(), - stm.relay_parent, - parachain_state->get(), - group, - candidate_hash); - } - - circulate_statement(stm.relay_parent, parachain_state->get(), stm.compact); - } - - void ParachainProcessorImpl::process_vstaging_statement( - const libp2p::peer::PeerId &peer_id, - const network::vstaging::StatementDistributionMessage &msg) { - BOOST_ASSERT(main_pool_handler_->isInCurrentThread()); - SL_TRACE( - logger_, "Incoming `StatementDistributionMessage`. (peer={})", peer_id); - - if (auto inner = - if_type( - msg)) { - handle_incoming_acknowledgement(peer_id, inner->get()); - } else if (auto manifest = - if_type( - msg)) { - handle_incoming_manifest(peer_id, manifest->get()); - } else if (auto stm = - if_type(msg)) { - handle_incoming_statement(peer_id, stm->get()); - } else { - SL_ERROR(logger_, "Skipped message."); - } - } - - void ParachainProcessorImpl::circulate_statement( - const RelayHash &relay_parent, - RelayParentState &relay_parent_state, - const IndexedAndSigned &statement) { - const auto &session_info = - relay_parent_state.per_session_state->value().session_info; - const auto &compact_statement = getPayload(statement); - const auto &candidate_hash = candidateHash(compact_statement); - const auto originator = statement.payload.ix; - const auto is_confirmed = candidates_.is_confirmed(candidate_hash); - - CHECK_OR_RET(relay_parent_state.local_validator); - enum DirectTargetKind : uint8_t { - Cluster, - Grid, - }; - - auto &local_validator = *relay_parent_state.local_validator; - auto targets = - [&]() -> std::vector> { - auto statement_group = - relay_parent_state.per_session_state->value().groups.byValidatorIndex( - originator); - - bool cluster_relevant = false; - std::vector> targets; - std::span all_cluster_targets; - - if (local_validator.active) { - auto &active = *local_validator.active; - cluster_relevant = - (statement_group && *statement_group == active.group); - if (is_confirmed && cluster_relevant) { - for (const auto v : active.cluster_tracker.targets()) { - if (active.cluster_tracker - .can_send(v, - originator, - network::vstaging::from(compact_statement)) - .has_error()) { - continue; - } - if (v == active.index) { - continue; - } - if (v >= session_info.discovery_keys.size()) { - continue; - } - targets.emplace_back(v, DirectTargetKind::Cluster); - } - } - all_cluster_targets = active.cluster_tracker.targets(); - } - - for (const auto v : local_validator.grid_tracker.direct_statement_targets( - relay_parent_state.per_session_state->value().groups, - originator, - compact_statement)) { - const auto can_use_grid = !cluster_relevant - || std::ranges::find(all_cluster_targets, v) - == all_cluster_targets.end(); - if (!can_use_grid) { - continue; - } - if (v >= session_info.discovery_keys.size()) { - continue; - } - targets.emplace_back(v, DirectTargetKind::Grid); - } - - return targets; - }(); - - std::vector> - statement_to_peers; - for (const auto &[target, kind] : targets) { - auto peer = query_audi_->get(session_info.discovery_keys[target]); - if (!peer) { - continue; - } - - auto peer_state = pm_->getPeerState(peer->id); - if (!peer_state) { - continue; - } - - if (!peer_state->get().knows_relay_parent(relay_parent)) { - continue; - } - - network::CollationVersion version = network::CollationVersion::VStaging; - if (peer_state->get().collation_version) { - version = *peer_state->get().collation_version; - } - - switch (kind) { - case Cluster: { - auto &active = *local_validator.active; - if (active.cluster_tracker - .can_send(target, - originator, - network::vstaging::from(compact_statement)) - .has_value()) { - active.cluster_tracker.note_sent( - target, originator, network::vstaging::from(compact_statement)); - statement_to_peers.emplace_back(peer->id, version); - } - } break; - case Grid: { - statement_to_peers.emplace_back(peer->id, version); - local_validator.grid_tracker.sent_or_received_direct_statement( - relay_parent_state.per_session_state->value().groups, - originator, - target, - compact_statement, - false); - } break; - } - } - - auto se = pm_->getStreamEngine(); - auto message_v2 = std::make_shared< - network::WireMessage>( - kagome::network::vstaging::ValidatorProtocolMessage{ - kagome::network::vstaging::StatementDistributionMessage{ - kagome::network::vstaging:: - StatementDistributionMessageStatement{ - .relay_parent = relay_parent, - .compact = statement, - }}}); - SL_TRACE( - logger_, - "Send statements to validators. (relay_parent={}, validators_count={})", - relay_parent, - statement_to_peers.size()); - for (const auto &[peer, version] : statement_to_peers) { - if (version == network::CollationVersion::VStaging) { - se->send(peer, router_->getValidationProtocolVStaging(), message_v2); - } else { - BOOST_ASSERT(false); - } - } - } - - void ParachainProcessorImpl::request_attested_candidate( - const libp2p::peer::PeerId &peer, - RelayParentState &relay_parent_state, - const RelayHash &relay_parent, - const CandidateHash &candidate_hash, - GroupIndex group_index) { - CHECK_OR_RET(relay_parent_state.local_validator); - auto &local_validator = *relay_parent_state.local_validator; - - const auto &session_info = - relay_parent_state.per_session_state->value().session_info; - - TRY_GET_OR_RET( - group, - relay_parent_state.per_session_state->value().groups.get(group_index)); - const auto seconding_limit = - relay_parent_state.prospective_parachains_mode->max_candidate_depth + 1; - - SL_TRACE(logger_, - "Form unwanted mask. (relay_parent={}, candidate_hash={})", - relay_parent, - candidate_hash); - network::vstaging::StatementFilter unwanted_mask(group->size()); - for (size_t i = 0; i < group->size(); ++i) { - const auto v = (*group)[i]; - if (relay_parent_state.statement_store->seconded_count(v) - >= seconding_limit) { - unwanted_mask.seconded_in_group.bits[i] = true; - } - } - - auto disabled_mask = relay_parent_state.disabled_bitmask(*group); - if (disabled_mask.bits.size() - > unwanted_mask.seconded_in_group.bits.size()) { - unwanted_mask.seconded_in_group.bits.resize(disabled_mask.bits.size()); - } - if (disabled_mask.bits.size() - > unwanted_mask.validated_in_group.bits.size()) { - unwanted_mask.validated_in_group.bits.resize(disabled_mask.bits.size()); - } - for (size_t i = 0; i < disabled_mask.bits.size(); ++i) { - unwanted_mask.seconded_in_group.bits[i] = - unwanted_mask.seconded_in_group.bits[i] || disabled_mask.bits[i]; - unwanted_mask.validated_in_group.bits[i] = - unwanted_mask.validated_in_group.bits[i] || disabled_mask.bits[i]; - } - - auto backing_threshold = [&]() -> std::optional { - auto bt = relay_parent_state.per_session_state->value() - .groups.get_size_and_backing_threshold(group_index); - return bt ? std::get<1>(*bt) : std::optional{}; - }(); - - SL_TRACE(logger_, - "Enumerate peers. (relay_parent={}, candidate_hash={})", - relay_parent, - candidate_hash); - std::optional target; - auto audi = query_audi_->get(peer); - if (!audi) { - SL_TRACE(logger_, - "No audi. (relay_parent={}, candidate_hash={})", - relay_parent, - candidate_hash); - return; - } - - ValidatorIndex validator_id = 0; - for (; validator_id < session_info.discovery_keys.size(); ++validator_id) { - if (session_info.discovery_keys[validator_id] == *audi) { - SL_TRACE(logger_, - "Captured validator. (relay_parent={}, candidate_hash={})", - relay_parent, - candidate_hash); - break; - } - } - - CHECK_OR_RET(validator_id < session_info.discovery_keys.size()); - auto filter = [&]() -> std::optional { - if (local_validator.active) { - if (local_validator.active->cluster_tracker.knows_candidate( - validator_id, candidate_hash)) { - return network::vstaging::StatementFilter( - local_validator.active->cluster_tracker.targets().size()); - } - } - - auto filter = local_validator.grid_tracker.advertised_statements( - validator_id, candidate_hash); - if (filter) { - return filter; - } - - SL_TRACE(logger_, - "No filter. (relay_parent={}, candidate_hash={})", - relay_parent, - candidate_hash); - return std::nullopt; - }(); - - CHECK_OR_RET(filter); - filter->mask_seconded(unwanted_mask.seconded_in_group); - filter->mask_valid(unwanted_mask.validated_in_group); - - if (!backing_threshold - || (filter->has_seconded() - && filter->backing_validators() >= *backing_threshold)) { - target.emplace(peer); - } else { - SL_TRACE( - logger_, - "Not pass backing threshold. (relay_parent={}, candidate_hash={})", - relay_parent, - candidate_hash); - return; - } - - if (!target) { - SL_TRACE(logger_, - "Target not found. (relay_parent={}, candidate_hash={})", - relay_parent, - candidate_hash); - return; - } - - SL_TRACE(logger_, - "Requesting. (peer={}, relay_parent={}, candidate_hash={})", - peer, - relay_parent, - candidate_hash); - router_->getFetchAttestedCandidateProtocol()->doRequest( - peer, - network::vstaging::AttestedCandidateRequest{ - .candidate_hash = candidate_hash, - .mask = unwanted_mask, - }, - [wptr{weak_from_this()}, - relay_parent{relay_parent}, - candidate_hash{candidate_hash}, - group_index{group_index}]( - outcome::result - r) mutable { - TRY_GET_OR_RET(self, wptr.lock()); - self->handleFetchedStatementResponse( - std::move(r), relay_parent, candidate_hash, group_index); - }); - } - - void ParachainProcessorImpl::handleFetchedStatementResponse( - outcome::result &&r, - const RelayHash &relay_parent, - const CandidateHash &candidate_hash, - GroupIndex group_index) { - REINVOKE(*main_pool_handler_, - handleFetchedStatementResponse, - std::move(r), - relay_parent, - candidate_hash, - group_index); - - if (r.has_error()) { - SL_INFO(logger_, - "Fetch attested candidate returned an error. (relay parent={}, " - "candidate={}, group index={}, error={})", - relay_parent, - candidate_hash, - group_index, - r.error()); - return; - } - - TRY_GET_OR_RET(parachain_state, tryGetStateByRelayParent(relay_parent)); - CHECK_OR_RET(parachain_state->get().statement_store); - - const network::vstaging::AttestedCandidateResponse &response = r.value(); - SL_INFO(logger_, - "Fetch attested candidate success. (relay parent={}, " - "candidate={}, group index={}, statements={})", - relay_parent, - candidate_hash, - group_index, - response.statements.size()); - for (const auto &statement : response.statements) { - parachain_state->get().statement_store->insert( - parachain_state->get().per_session_state->value().groups, - statement, - StatementOrigin::Remote); - } - - auto opt_post_confirmation = - candidates_.confirm_candidate(candidate_hash, - response.candidate_receipt, - response.persisted_validation_data, - group_index, - hasher_); - if (!opt_post_confirmation) { - SL_WARN(logger_, - "Candidate re-confirmed by request/response: logic error. (relay " - "parent={}, candidate={})", - relay_parent, - candidate_hash); - return; - } - - auto &post_confirmation = *opt_post_confirmation; - apply_post_confirmation(post_confirmation); - - auto opt_confirmed = candidates_.get_confirmed(candidate_hash); - BOOST_ASSERT(opt_confirmed); - - if (!opt_confirmed->get().is_importable(std::nullopt)) { - SL_INFO(logger_, - "Not importable. (relay parent={}, " - "candidate={}, group index={})", - relay_parent, - candidate_hash, - group_index); - return; - } - - const auto &groups = - parachain_state->get().per_session_state->value().groups; - auto it = groups.groups.find(group_index); - if (it == groups.groups.end()) { - SL_WARN(logger_, - "Group was not found. (relay parent={}, candidate={}, group " - "index={})", - relay_parent, - candidate_hash, - group_index); - return; - } - - SL_INFO(logger_, - "Send fresh statements. (relay parent={}, " - "candidate={})", - relay_parent, - candidate_hash); - send_backing_fresh_statements(opt_confirmed->get(), - relay_parent, - parachain_state->get(), - it->second, - candidate_hash); - } - - void ParachainProcessorImpl::new_confirmed_candidate_fragment_chain_updates( - const HypotheticalCandidate &candidate) { - fragment_chain_update_inner(std::nullopt, std::nullopt, {candidate}); - } - - void ParachainProcessorImpl::new_leaf_fragment_chain_updates( - const Hash &leaf_hash) { - fragment_chain_update_inner({leaf_hash}, std::nullopt, std::nullopt); - } - - void ParachainProcessorImpl:: - prospective_backed_notification_fragment_chain_updates( - ParachainId para_id, const Hash ¶_head) { - std::pair, ParachainId> p{{para_head}, - para_id}; - fragment_chain_update_inner(std::nullopt, p, std::nullopt); - } - - void ParachainProcessorImpl::fragment_chain_update_inner( - std::optional> active_leaf_hash, - std::optional, ParachainId>> - required_parent_info, - std::optional> - known_hypotheticals) { - std::vector hypotheticals; - if (!known_hypotheticals) { - hypotheticals = candidates_.frontier_hypotheticals(required_parent_info); - } else { - hypotheticals.emplace_back(known_hypotheticals->get()); - } - - auto frontier = - prospective_parachains_->answer_hypothetical_membership_request( - hypotheticals, active_leaf_hash); - for (const auto &[hypo, membership] : frontier) { - if (membership.empty()) { - continue; - } - - for (const auto &leaf_hash : membership) { - candidates_.note_importable_under(hypo, leaf_hash); - } - - if (auto c = if_type(hypo)) { - auto confirmed_candidate = - candidates_.get_confirmed(c->get().candidate_hash); - auto prs = - tryGetStateByRelayParent(c->get().receipt.descriptor.relay_parent); - - if (prs && confirmed_candidate) { - const auto group_index = - group_for_para(prs->get().availability_cores, - prs->get().group_rotation_info, - c->get().receipt.descriptor.para_id); - - const auto &session_info = - prs->get().per_session_state->value().session_info; - if (!group_index - || *group_index >= session_info.validator_groups.size()) { - return; - } - - const auto &group = session_info.validator_groups[*group_index]; - send_backing_fresh_statements( - *confirmed_candidate, - c->get().receipt.descriptor.relay_parent, - prs->get(), - group, - c->get().candidate_hash); - } - } - } - } - - /// TODO(iceseer): https://github.com/qdrvm/kagome/issues/2133 - /// TODO(iceseer): do remove - std::optional ParachainProcessorImpl::group_for_para( - const std::vector &availability_cores, - const runtime::GroupDescriptor &group_rotation_info, - ParachainId para_id) const { - std::optional core_index; - for (CoreIndex i = 0; i < availability_cores.size(); ++i) { - const auto c = visit_in_place( - availability_cores[i], - [](const runtime::OccupiedCore &core) -> std::optional { - return core.candidate_descriptor.para_id; - }, - [](const runtime::ScheduledCore &core) -> std::optional { - return core.para_id; - }, - [](const auto &) -> std::optional { - return std::nullopt; - }); - - if (c && *c == para_id) { - core_index = i; - break; - } - } - - if (!core_index) { - return std::nullopt; - } - return group_rotation_info.groupForCore(*core_index, - availability_cores.size()); - } - - void ParachainProcessorImpl::send_cluster_candidate_statements( - const CandidateHash &candidate_hash, const RelayHash &relay_parent) { - TRY_GET_OR_RET(relay_parent_state, tryGetStateByRelayParent(relay_parent)); - TRY_GET_OR_RET(local_group, relay_parent_state->get().our_group); - TRY_GET_OR_RET( - group, - relay_parent_state->get().per_session_state->value().groups.get( - *local_group)); - - auto group_size = group->size(); - relay_parent_state->get().statement_store->groupStatements( - *group, - candidate_hash, - network::vstaging::StatementFilter(group_size, true), - [&](const IndexedAndSigned - &statement) { - circulate_statement( - relay_parent, relay_parent_state->get(), statement); - }); - } - - void ParachainProcessorImpl::apply_post_confirmation( - const PostConfirmation &post_confirmation) { - const auto candidate_hash = candidateHash(post_confirmation.hypothetical); - send_cluster_candidate_statements( - candidate_hash, relayParent(post_confirmation.hypothetical)); - - new_confirmed_candidate_fragment_chain_updates( - post_confirmation.hypothetical); - } - - void ParachainProcessorImpl::send_backing_fresh_statements( - const ConfirmedCandidate &confirmed, - const RelayHash &relay_parent, - ParachainProcessorImpl::RelayParentState &per_relay_parent, - const std::vector &group, - const CandidateHash &candidate_hash) { - CHECK_OR_RET(per_relay_parent.statement_store); - std::vector> - imported; - per_relay_parent.statement_store->fresh_statements_for_backing( - group, - candidate_hash, - [&](const IndexedAndSigned - &statement) { - const auto &v = statement.payload.ix; - const auto &compact = getPayload(statement); - imported.emplace_back(v, compact); - - SignedFullStatementWithPVD carrying_pvd{ - .payload = - { - .payload = visit_in_place( - compact.inner_value, - [&](const network::vstaging::SecondedCandidateHash &) - -> StatementWithPVD { - return StatementWithPVDSeconded{ - .committed_receipt = confirmed.receipt, - .pvd = confirmed.persisted_validation_data, - }; - }, - [](const network::vstaging::ValidCandidateHash &val) - -> StatementWithPVD { - return StatementWithPVDValid{ - .candidate_hash = val.hash, - }; - }, - [](const auto &) -> StatementWithPVD { - UNREACHABLE; - }), - .ix = statement.payload.ix, - }, - .signature = statement.signature, - }; + SL_TRACE(logger_, "Send to peer from any. (peer={})", peer); + se->send(peer, protocol, message); + } + }; - main_pool_handler_->execute( - [wself{weak_from_this()}, - relay_parent{relay_parent}, - carrying_pvd{std::move(carrying_pvd)}]() { - TRY_GET_OR_RET(self, wself.lock()); + for (const network::VersionedValidatorProtocolMessage &msg : messages) { + visit_in_place( + msg, + [&](const kagome::network::vstaging::ValidatorProtocolMessage &m) { + make_send(m, router_->getValidationProtocolVStaging()); + }, + [&](const kagome::network::ValidatorProtocolMessage &m) { + make_send(m, router_->getValidationProtocol()); + }); + } + } - SL_TRACE(self->logger_, "Handle statement {}", relay_parent); - self->handleStatement(relay_parent, carrying_pvd); - }); - }); + void ParachainProcessorImpl::process_vstaging_statement( + const libp2p::peer::PeerId &peer_id, + const network::vstaging::StatementDistributionMessage &msg) { + BOOST_ASSERT(main_pool_handler_->isInCurrentThread()); + SL_TRACE( + logger_, "Incoming `StatementDistributionMessage`. (peer={})", peer_id); - for (const auto &[v, s] : imported) { - per_relay_parent.statement_store->note_known_by_backing(v, s); + if (auto inner = + if_type( + msg)) { + statement_distribution->handle_incoming_acknowledgement(peer_id, + inner->get()); + } else if (auto manifest = + if_type( + msg)) { + statement_distribution->handle_incoming_manifest(peer_id, + manifest->get()); + } else if (auto stm = + if_type(msg)) { + statement_distribution->handle_incoming_statement(peer_id, stm->get()); + } else { + SL_ERROR(logger_, "Skipped message."); } } @@ -3180,159 +1443,6 @@ namespace kagome::parachain { } } - outcome::result - ParachainProcessorImpl::OnFetchAttestedCandidateRequest( - const network::vstaging::AttestedCandidateRequest &request, - const libp2p::peer::PeerId &peer_id) { - auto confirmed = candidates_.get_confirmed(request.candidate_hash); - if (!confirmed) { - return Error::NOT_CONFIRMED; - } - - OUTCOME_TRY(relay_parent_state, - getStateByRelayParent(confirmed->get().relay_parent())); - auto &local_validator = relay_parent_state.get().local_validator; - if (!local_validator) { - return Error::NOT_A_VALIDATOR; - } - BOOST_ASSERT(relay_parent_state.get().statement_store); - - const auto &session_info = - relay_parent_state.get().per_session_state->value().session_info; - const auto &groups = - relay_parent_state.get().per_session_state->value().groups; - auto group = groups.get(confirmed->get().group_index()); - if (!group) { - SL_ERROR(logger_, - "Unexpected array bound for groups. (relay parent={})", - confirmed->get().relay_parent()); - return Error::OUT_OF_BOUND; - } - - const auto group_size = group->size(); - auto &mask = request.mask; - if (mask.seconded_in_group.bits.size() != group_size - || mask.validated_in_group.bits.size() != group_size) { - return Error::INCORRECT_BITFIELD_SIZE; - } - - auto &active = local_validator->active; - std::optional validator_id; - bool is_cluster = false; - [&] { - auto audi = query_audi_->get(peer_id); - if (not audi.has_value()) { - SL_TRACE(logger_, "No audi. (peer={})", peer_id); - return; - } - - ValidatorIndex v = 0; - for (; v < session_info.discovery_keys.size(); ++v) { - if (session_info.discovery_keys[v] == audi.value()) { - SL_TRACE(logger_, - "Captured validator. (relay_parent={}, candidate_hash={})", - confirmed->get().relay_parent(), - request.candidate_hash); - break; - } - } - - if (v >= session_info.discovery_keys.size()) { - return; - } - - if (active - and active->cluster_tracker.can_request(v, request.candidate_hash)) { - validator_id = v; - is_cluster = true; - - } else if (local_validator->grid_tracker.can_request( - v, request.candidate_hash)) { - validator_id = v; - } - }(); - - if (!validator_id) { - return Error::OUT_OF_BOUND; - } - - auto init_with_not = [](scale::BitVec &dst, const scale::BitVec &src) { - dst.bits.reserve(src.bits.size()); - for (const auto i : src.bits) { - dst.bits.emplace_back(!i); - } - }; - - network::vstaging::StatementFilter and_mask; - init_with_not(and_mask.seconded_in_group, request.mask.seconded_in_group); - init_with_not(and_mask.validated_in_group, request.mask.validated_in_group); - - std::vector> - statements; - network::vstaging::StatementFilter sent_filter(group_size); - relay_parent_state.get().statement_store->groupStatements( - *group, - request.candidate_hash, - and_mask, - [&](const IndexedAndSigned - &statement) { - for (size_t ix = 0; ix < group->size(); ++ix) { - if ((*group)[ix] == statement.payload.ix) { - visit_in_place( - getPayload(statement).inner_value, - [&](const network::vstaging::SecondedCandidateHash &) { - sent_filter.seconded_in_group.bits[ix] = true; - }, - [&](const network::vstaging::ValidCandidateHash &) { - sent_filter.validated_in_group.bits[ix] = true; - }, - [&](const auto &) {}); - } - } - statements.emplace_back(statement); - }); - - if (!is_cluster) { - auto threshold = std::get<1>(*groups.get_size_and_backing_threshold( - confirmed->get().group_index())); - const auto seconded_and_sufficient = - (sent_filter.has_seconded() - && sent_filter.backing_validators() >= threshold); - if (!seconded_and_sufficient) { - SL_INFO(logger_, - "Dropping a request from a grid peer because the backing " - "threshold is no longer met. (relay_parent={}, " - "candidate_hash={}, group_index={})", - confirmed->get().relay_parent(), - request.candidate_hash, - confirmed->get().group_index()); - return Error::THRESHOLD_LIMIT_REACHED; - } - } - - for (const auto &statement : statements) { - if (is_cluster) { - active->cluster_tracker.note_sent( - *validator_id, - statement.payload.ix, - network::vstaging::from(getPayload(statement))); - } else { - local_validator->grid_tracker.sent_or_received_direct_statement( - groups, - statement.payload.ix, - *validator_id, - getPayload(statement), - false); - } - } - - return network::vstaging::AttestedCandidateResponse{ - .candidate_receipt = confirmed->get().receipt, - .persisted_validation_data = confirmed->get().persisted_validation_data, - .statements = std::move(statements), - }; - } - outcome::result ParachainProcessorImpl::OnFetchChunkRequest( const network::FetchChunkRequest &request) { @@ -3400,7 +1510,7 @@ namespace kagome::parachain { void ParachainProcessorImpl::handleStatement( const primitives::BlockHash &relay_parent, const SignedFullStatementWithPVD &statement) { - BOOST_ASSERT(main_pool_handler_->isInCurrentThread()); + REINVOKE(*main_pool_handler_, handleStatement, relay_parent, statement); TRY_GET_OR_RET(opt_parachain_state, tryGetStateByRelayParent(relay_parent)); auto ¶chain_state = opt_parachain_state->get(); @@ -3408,6 +1518,7 @@ namespace kagome::parachain { const auto &assigned_core = parachain_state.assigned_core; auto &fallbacks = parachain_state.fallbacks; auto &awaiting_validation = parachain_state.awaiting_validation; + auto &table_context = parachain_state.table_context; auto res = importStatement(relay_parent, statement, parachain_state); if (res.has_error()) { @@ -3463,8 +1574,11 @@ namespace kagome::parachain { if (it == fallbacks.end()) { return std::nullopt; } - if (!parachain_state.our_index - || *parachain_state.our_index == statement.payload.ix) { + + const auto our_index = utils::map( + table_context.validator, + [](const auto &signer) { return signer.validatorIndex(); }); + if (our_index && *our_index == statement.payload.ix) { return std::nullopt; } if (awaiting_validation.find(val.candidate_hash) @@ -3510,221 +1624,6 @@ namespace kagome::parachain { relayParentState.prospective_parachains_mode.has_value()); } - void ParachainProcessorImpl::provide_candidate_to_grid( - const CandidateHash &candidate_hash, - RelayParentState &relay_parent_state, - const ConfirmedCandidate &confirmed_candidate, - const runtime::SessionInfo &session_info) { - CHECK_OR_RET(relay_parent_state.local_validator); - auto &local_validator = *relay_parent_state.local_validator; - - const auto relay_parent = confirmed_candidate.relay_parent(); - const auto group_index = confirmed_candidate.group_index(); - - if (!relay_parent_state.per_session_state->value().grid_view) { - SL_TRACE(logger_, - "Cannot handle backable candidate due to lack of topology. " - "(candidate={}, relay_parent={})", - candidate_hash, - relay_parent); - return; - } - - const auto &grid_view = - *relay_parent_state.per_session_state->value().grid_view; - const auto group = - relay_parent_state.per_session_state->value().groups.get(group_index); - if (!group) { - SL_TRACE(logger_, - "Handled backed candidate with unknown group? (candidate={}, " - "relay_parent={}, group_index={})", - candidate_hash, - relay_parent, - group_index); - return; - } - - const auto group_size = group->size(); - auto filter = local_knowledge_filter(group_size, - group_index, - candidate_hash, - *relay_parent_state.statement_store); - - auto actions = local_validator.grid_tracker.add_backed_candidate( - grid_view, candidate_hash, group_index, filter); - - network::vstaging::BackedCandidateManifest manifest{ - .relay_parent = relay_parent, - .candidate_hash = candidate_hash, - .group_index = group_index, - .para_id = confirmed_candidate.para_id(), - .parent_head_data_hash = confirmed_candidate.parent_head_data_hash(), - .statement_knowledge = filter}; - - network::vstaging::BackedCandidateAcknowledgement acknowledgement{ - .candidate_hash = candidate_hash, .statement_knowledge = filter}; - - std::vector> - manifest_peers; - std::vector> - ack_peers; - std::deque, - network::VersionedValidatorProtocolMessage>> - post_statements; - - for (const auto &[v, action] : actions) { - auto peer_opt = query_audi_->get(session_info.discovery_keys[v]); - if (!peer_opt) { - SL_TRACE(logger_, - "No peer info. (relay_parent={}, validator_index={}, " - "candidate_hash={})", - relay_parent, - v, - candidate_hash); - continue; - } - - auto peer_state = pm_->getPeerState(peer_opt->id); - if (!peer_state) { - SL_TRACE(logger_, - "No peer state. (relay_parent={}, peer={}, candidate_hash={})", - relay_parent, - peer_opt->id, - candidate_hash); - continue; - } - - if (!peer_state->get().knows_relay_parent(relay_parent)) { - SL_TRACE(logger_, - "Peer doesn't know relay parent. (relay_parent={}, peer={}, " - "candidate_hash={})", - relay_parent, - peer_opt->id, - candidate_hash); - continue; - } - - switch (action) { - case grid::ManifestKind::Full: { - SL_TRACE(logger_, "Full manifest -> {}", v); - manifest_peers.emplace_back(peer_opt->id, - network::CollationVersion::VStaging); - } break; - case grid::ManifestKind::Acknowledgement: { - SL_TRACE(logger_, "Ack manifest -> {}", v); - ack_peers.emplace_back(peer_opt->id, - network::CollationVersion::VStaging); - } break; - } - - local_validator.grid_tracker.manifest_sent_to( - relay_parent_state.per_session_state->value().groups, - v, - candidate_hash, - filter); - - auto msgs = post_acknowledgement_statement_messages( - v, - relay_parent, - local_validator.grid_tracker, - *relay_parent_state.statement_store, - relay_parent_state.per_session_state->value().groups, - group_index, - candidate_hash, - peer_opt->id, - network::CollationVersion::VStaging); - - for (auto &msg : msgs) { - post_statements.emplace_back( - std::vector{peer_opt->id}, std::move(msg)); - } - } - - auto se = pm_->getStreamEngine(); - if (!manifest_peers.empty()) { - SL_TRACE(logger_, - "Sending manifest to v2 peers. (candidate_hash={}, " - "local_validator={}, n_peers={})", - candidate_hash, - *relay_parent_state.our_index, - manifest_peers.size()); - auto message = std::make_shared< - network::WireMessage>( - kagome::network::vstaging::ValidatorProtocolMessage{ - kagome::network::vstaging::StatementDistributionMessage{ - manifest}}); - for (const auto &[p, _] : manifest_peers) { - se->send(p, router_->getValidationProtocolVStaging(), message); - } - } - - if (!ack_peers.empty()) { - SL_TRACE(logger_, - "Sending acknowledgement to v2 peers. (candidate_hash={}, " - "local_validator={}, n_peers={})", - candidate_hash, - *relay_parent_state.our_index, - ack_peers.size()); - auto message = std::make_shared< - network::WireMessage>( - kagome::network::vstaging::ValidatorProtocolMessage{ - kagome::network::vstaging::StatementDistributionMessage{ - acknowledgement}}); - for (const auto &[p, _] : ack_peers) { - se->send(p, router_->getValidationProtocolVStaging(), message); - } - } - - if (!post_statements.empty()) { - SL_TRACE(logger_, - "Sending statements to v2 peers. (candidate_hash={}, " - "local_validator={}, n_peers={})", - candidate_hash, - *relay_parent_state.our_index, - post_statements.size()); - - for (auto &[peers, msg] : post_statements) { - if (auto m = - if_type(msg)) { - auto message = std::make_shared>( - std::move(m->get())); - for (const auto &p : peers) { - se->send(p, router_->getValidationProtocolVStaging(), message); - } - } else { - assert(false); - } - } - } - } - - void ParachainProcessorImpl::statementDistributionBackedCandidate( - const CandidateHash &candidate_hash) { - auto confirmed_opt = candidates_.get_confirmed(candidate_hash); - if (!confirmed_opt) { - SL_TRACE(logger_, - "Received backed candidate notification for unknown or " - "unconfirmed. (candidate_hash={})", - candidate_hash); - return; - } - const auto &confirmed = confirmed_opt->get(); - - const auto relay_parent = confirmed.relay_parent(); - TRY_GET_OR_RET(relay_parent_state_opt, - tryGetStateByRelayParent(relay_parent)); - BOOST_ASSERT(relay_parent_state_opt->get().statement_store); - - const auto &session_info = - relay_parent_state_opt->get().per_session_state->value().session_info; - provide_candidate_to_grid( - candidate_hash, relay_parent_state_opt->get(), confirmed, session_info); - - prospective_backed_notification_fragment_chain_updates( - confirmed.para_id(), confirmed.para_head()); - } - outcome::result ParachainProcessorImpl::get_block_number_under_construction( const RelayHash &relay_parent) const { @@ -4092,7 +1991,7 @@ namespace kagome::parachain { const SignedFullStatementWithPVD &statement, ParachainProcessorImpl::RelayParentState &rp_state) { const CandidateHash candidate_hash = - candidateHashFrom(parachain::getPayload(statement)); + candidateHashFrom(parachain::getPayload(statement), hasher_); SL_TRACE(logger_, "Importing statement.(relay_parent={}, validator_index={}, " @@ -4164,7 +2063,7 @@ namespace kagome::parachain { const std::vector &cores, const SignedFullStatementWithPVD &statement) { const auto &compact_statement = getPayload(statement); - const auto candidate_hash = candidateHashFrom(compact_statement); + const auto candidate_hash = candidateHashFrom(compact_statement, hasher_); const auto n_cores = cores.size(); SL_TRACE( @@ -4257,74 +2156,8 @@ namespace kagome::parachain { return core_index; } - void ParachainProcessorImpl::unblockAdvertisements( - ParachainProcessorImpl::RelayParentState &rp_state, - ParachainId para_id, - const Hash ¶_head) { - std::optional> unblocked{}; - auto it = our_current_state_.blocked_advertisements.find(para_id); - if (it != our_current_state_.blocked_advertisements.end()) { - auto i = it->second.find(para_head); - if (i != it->second.end()) { - unblocked = std::move(i->second); - it->second.erase(i); - } - } - - if (unblocked) { - requestUnblockedCollations( - {{para_id, {{para_head, std::move(*unblocked)}}}}); - } - } - - void ParachainProcessorImpl::requestUnblockedCollations( - std::unordered_map< - ParachainId, - std::unordered_map>> - &&blocked) { - for (const auto &[para_id, v] : blocked) { - for (const auto &[para_head, blocked_tmp] : v) { - std::vector blocked_vec; - for (const auto &blocked : blocked_tmp) { - const auto is_seconding_allowed = - canSecond(para_id, - blocked.candidate_relay_parent, - blocked.candidate_hash, - para_head); - if (is_seconding_allowed) { - auto result = - enqueueCollation(blocked.candidate_relay_parent, - para_id, - blocked.peer_id, - blocked.collator_id, - std::make_optional(std::make_pair( - blocked.candidate_hash, para_head))); - if (result.has_error()) { - SL_DEBUG(logger_, - "Enqueue collation failed.(candidate={}, para id={}, " - "relay_parent={}, para_head={}, peer_id={})", - blocked.candidate_hash, - para_id, - blocked.candidate_relay_parent, - para_head, - blocked.peer_id); - } - } else { - blocked_vec.emplace_back(blocked); - } - } - - if (!blocked_vec.empty()) { - our_current_state_.blocked_advertisements[para_id][para_head] = - std::move(blocked_vec); - } - } - } - } - template - outcome::result< - std::optional> + outcome::result> ParachainProcessorImpl::sign_import_and_distribute_statement( ParachainProcessorImpl::RelayParentState &rp_state, const ValidateAndSecondResult &validation_result) { @@ -4368,8 +2201,8 @@ namespace kagome::parachain { OUTCOME_TRY( summary, importStatement(validation_result.relay_parent, stm, rp_state)); - share_local_statement_vstaging( - rp_state, validation_result.relay_parent, stm); + statement_distribution->share_local_statement( + validation_result.relay_parent, stm); post_import_statement_actions( validation_result.relay_parent, rp_state, summary); @@ -4410,9 +2243,8 @@ namespace kagome::parachain { if (rp_state.prospective_parachains_mode) { prospective_parachains_->candidate_backed(para_id, summary->candidate); - unblockAdvertisements( - rp_state, para_id, backed->candidate.descriptor.para_head_hash); - statementDistributionBackedCandidate(summary->candidate); + statement_distribution->handle_backed_candidate_message( + summary->candidate); } else { backing_store_->add(relay_parent, std::move(*backed)); } @@ -4446,7 +2278,7 @@ namespace kagome::parachain { return std::nullopt; } - if (!parachain_state->get().our_index) { + if (!parachain_state->get().table_context.validator) { logger_->warn("We are not validators or we have no validator index."); return std::nullopt; } @@ -4457,13 +2289,11 @@ namespace kagome::parachain { network::CandidateState{network::CommittedCandidateReceipt{ .descriptor = validation_result.candidate.descriptor, .commitments = *validation_result.commitments}}}, - *parachain_state->get().our_index, parachain_state->get()); } else if constexpr (kStatementType == StatementType::kValid) { return createAndSignStatementFromPayload( network::Statement{network::CandidateState{ validation_result.candidate.hash(*hasher_)}}, - *parachain_state->get().our_index, parachain_state->get()); } } @@ -4471,9 +2301,7 @@ namespace kagome::parachain { template std::optional ParachainProcessorImpl::createAndSignStatementFromPayload( - T &&payload, - ValidatorIndex validator_ix, - RelayParentState ¶chain_state) { + T &&payload, RelayParentState ¶chain_state) { /// TODO(iceseer): /// https://github.com/paritytech/polkadot/blob/master/primitives/src/v2/mod.rs#L1535-L1545 auto sign_result = @@ -4505,9 +2333,9 @@ namespace kagome::parachain { CHECK_OR_RET(canProcessParachains().has_value()); SL_DEBUG(logger_, - "Send my view.(peer={}, protocol={})", - peer_id, - protocol->protocolName()); + "Send my view.(peer={}, protocol={})", + peer_id, + protocol->protocolName()); pm_->getStreamEngine()->send( peer_id, protocol, @@ -4658,6 +2486,10 @@ namespace kagome::parachain { REINVOKE_ONCE( *main_pool_handler_, notifyInvalid, parent, candidate_receipt); + our_current_state_.validator_side.blocked_from_seconding.erase( + BlockedCollationId(candidate_receipt.descriptor.para_id, + candidate_receipt.descriptor.para_head_hash)); + auto fetched_collation = network::FetchedCollation::from(candidate_receipt, *hasher_); const auto &candidate_hash = fetched_collation.candidate_hash; @@ -4702,6 +2534,10 @@ namespace kagome::parachain { return; } + auto output_head_data = + seconded->get().committed_receipt.commitments.para_head; + auto output_head_data_hash = + seconded->get().committed_receipt.descriptor.para_head_hash; auto fetched_collation = network::FetchedCollation::from( seconded->get().committed_receipt.to_plain(*hasher_), *hasher_); auto it = our_current_state_.validator_side.fetched_candidates.find( @@ -4737,6 +2573,9 @@ namespace kagome::parachain { rp_state->get().collations.note_seconded(); } + second_unblocked_collations( + fetched_collation.para_id, output_head_data, output_head_data_hash); + const auto maybe_candidate_hash = utils::map( prospective_candidate, [](const auto &v) { return v.candidate_hash; }); @@ -4838,157 +2677,6 @@ namespace kagome::parachain { notifySeconded(validation_result.relay_parent, stmt); } - void ParachainProcessorImpl::share_local_statement_v1( - RelayParentState &per_relay_parent, - const primitives::BlockHash &relay_parent, - const SignedFullStatementWithPVD &statement) { - send_to_validators_group( - relay_parent, - {network::ValidatorProtocolMessage{ - network::StatementDistributionMessage{network::Seconded{ - .relay_parent = relay_parent, - .statement = network::SignedStatement{ - .payload = - { - .payload = visit_in_place( - parachain::getPayload(statement), - [&](const StatementWithPVDSeconded &val) { - return network::CandidateState{ - val.committed_receipt}; - }, - [&](const StatementWithPVDValid &val) { - return network::CandidateState{ - val.candidate_hash}; - }), - .ix = statement.payload.ix, - }, - .signature = statement.signature, - }}}}}); - } - - void ParachainProcessorImpl::share_local_statement_vstaging( - RelayParentState &per_relay_parent, - const primitives::BlockHash &relay_parent, - const SignedFullStatementWithPVD &statement) { - const CandidateHash candidate_hash = - candidateHashFrom(getPayload(statement)); - SL_TRACE(logger_, - "Sharing statement. (relay parent={}, candidate hash={}, " - "our_index={}, statement_ix={})", - relay_parent, - candidate_hash, - *per_relay_parent.our_index, - statement.payload.ix); - - BOOST_ASSERT(per_relay_parent.our_index); - - const Groups &groups = per_relay_parent.per_session_state->value().groups; - const std::optional &local_assignment = - per_relay_parent.assigned_para; - const network::ValidatorIndex local_index = *per_relay_parent.our_index; - const auto local_group_opt = groups.byValidatorIndex(local_index); - const GroupIndex local_group = *local_group_opt; - - std::optional> expected = visit_in_place( - getPayload(statement), - [&](const StatementWithPVDSeconded &v) - -> std::optional> { - return std::make_pair(v.committed_receipt.descriptor.para_id, - v.committed_receipt.descriptor.relay_parent); - }, - [&](const StatementWithPVDValid &v) - -> std::optional> { - if (auto p = candidates_.get_confirmed(v.candidate_hash)) { - return std::make_pair(p->get().para_id(), p->get().relay_parent()); - } - return std::nullopt; - }); - const bool is_seconded = - is_type(getPayload(statement)); - - if (!expected) { - SL_ERROR( - logger_, "Invalid share statement. (relay parent={})", relay_parent); - return; - } - const auto &[expected_para, expected_relay_parent] = *expected; - - if (local_index != statement.payload.ix) { - SL_ERROR(logger_, - "Invalid share statement because of validator index. (relay " - "parent={})", - relay_parent); - return; - } - - BOOST_ASSERT(per_relay_parent.statement_store); - BOOST_ASSERT(per_relay_parent.prospective_parachains_mode); - - const auto seconding_limit = - per_relay_parent.prospective_parachains_mode->max_candidate_depth + 1; - if (is_seconded - && per_relay_parent.statement_store->seconded_count(local_index) - == seconding_limit) { - SL_WARN( - logger_, - "Local node has issued too many `Seconded` statements. (limit={})", - seconding_limit); - return; - } - - if (!local_assignment || *local_assignment != expected_para - || relay_parent != expected_relay_parent) { - SL_ERROR( - logger_, - "Invalid share statement because local assignment. (relay parent={})", - relay_parent); - return; - } - - IndexedAndSigned compact_statement = - signed_to_compact(statement); - std::optional post_confirmation; - if (auto s = - if_type(getPayload(statement))) { - post_confirmation = - candidates_.confirm_candidate(candidate_hash, - s->get().committed_receipt, - s->get().pvd, - local_group, - hasher_); - } - - if (auto r = per_relay_parent.statement_store->insert( - groups, compact_statement, StatementOrigin::Local); - !r || !*r) { - SL_ERROR(logger_, - "Invalid share statement because statement store insertion " - "failed. (relay parent={})", - relay_parent); - return; - } - - if (per_relay_parent.local_validator - && per_relay_parent.local_validator->active) { - per_relay_parent.local_validator->active->cluster_tracker.note_issued( - local_index, network::vstaging::from(getPayload(compact_statement))); - } - - if (per_relay_parent.per_session_state->value().grid_view) { - auto &l = *per_relay_parent.local_validator; - l.grid_tracker.learned_fresh_statement( - per_relay_parent.per_session_state->value().groups, - *per_relay_parent.per_session_state->value().grid_view, - local_index, - getPayload(compact_statement)); - } - - circulate_statement(relay_parent, per_relay_parent, compact_statement); - if (post_confirmation) { - apply_post_confirmation(*post_confirmation); - } - } - outcome::result> ParachainProcessorImpl::validateErasureCoding( const runtime::AvailableData &validating_data, size_t n_validators) { @@ -5098,8 +2786,6 @@ namespace kagome::parachain { .validation_data = std::move(data), }; - auto measure2 = - std::make_shared("===> EC validation", self->logger_); auto chunks_res = self->validateErasureCoding(available_data, n_validators); if (chunks_res.has_error()) { @@ -5108,7 +2794,6 @@ namespace kagome::parachain { chunks_res.error()); return; } - measure2.reset(); auto &chunks = chunks_res.value(); self->notifyAvailableData(std::move(chunks), @@ -5271,11 +2956,12 @@ namespace kagome::parachain { return Error::UNDECLARED_COLLATOR; } - if (!isRelayParentInImplicitView(on_relay_parent, - relay_parent_mode, - *our_current_state_.implicit_view, - our_current_state_.active_leaves, - peer_data.collator_state->para_id)) { + if (!isRelayParentInImplicitView( + on_relay_parent, + relay_parent_mode, + *our_current_state_.implicit_view, + our_current_state_.validator_side.active_leaves, + peer_data.collator_state->para_id)) { SL_TRACE(logger_, "Out of view. (relay_parent={})", on_relay_parent); return Error::OUT_OF_VIEW; } @@ -5308,7 +2994,7 @@ namespace kagome::parachain { peer_data.collator_state->para_id); } - outcome::result ParachainProcessorImpl::kick_off_seconding( + outcome::result ParachainProcessorImpl::kick_off_seconding( network::PendingCollationFetch &&pending_collation_fetch) { BOOST_ASSERT(main_pool_handler_->isInCurrentThread()); @@ -5337,7 +3023,10 @@ namespace kagome::parachain { per_relay_parent.get().prospective_parachains_mode.has_value(); std::optional maybe_pvd; - std::optional> maybe_parent_head_and_hash; + std::optional maybe_parent_head_hash; + std::optional &maybe_parent_head = + pending_collation_fetch.maybe_parent_head_data; + if (is_collator_v2 && have_prospective_candidate && async_backing_en) { OUTCOME_TRY(pvd, requestProspectiveValidationData( @@ -5349,8 +3038,7 @@ namespace kagome::parachain { maybe_pvd = pvd; if (pending_collation_fetch.maybe_parent_head_data) { - maybe_parent_head_and_hash.emplace( - *pending_collation_fetch.maybe_parent_head_data, + maybe_parent_head_hash.emplace( collation_event.pending_collation.prospective_candidate ->parent_head_data_hash); } @@ -5362,32 +3050,59 @@ namespace kagome::parachain { pending_collation_fetch.candidate_receipt.descriptor.relay_parent, pending_collation_fetch.candidate_receipt.descriptor.para_id)); maybe_pvd = pvd; - maybe_parent_head_and_hash = std::nullopt; + maybe_parent_head_hash = std::nullopt; + } else { + return outcome::success(false); + } + + std::optional> pvd; + if (maybe_pvd) { + pvd = *maybe_pvd; + } else if (!maybe_pvd && !maybe_parent_head && maybe_parent_head_hash) { + const network::PendingCollationFetch blocked_collation{ + .collation_event = std::move(collation_event), + .candidate_receipt = pending_collation_fetch.candidate_receipt, + .pov = std::move(pending_collation_fetch.pov), + .maybe_parent_head_data = std::nullopt, + }; + SL_TRACE(logger_, + "Collation having parent head data hash {} is blocked from " + "seconding. Waiting on its parent to be validated. " + "(candidate_hash={}, relay_parent={})", + *maybe_parent_head_hash, + blocked_collation.candidate_receipt.hash(*hasher_), + blocked_collation.candidate_receipt.descriptor.relay_parent); + our_current_state_.validator_side + .blocked_from_seconding[BlockedCollationId( + blocked_collation.candidate_receipt.descriptor.para_id, + *maybe_parent_head_hash)] + .emplace_back(blocked_collation); + + return outcome::success(false); } else { - return outcome::success(); - } - - if (!maybe_pvd) { return Error::PERSISTED_VALIDATION_DATA_NOT_FOUND; } - auto pvd{std::move(*maybe_pvd)}; OUTCOME_TRY(fetched_collation_sanity_check( collation_event.pending_collation, pending_collation_fetch.candidate_receipt, - pvd, - maybe_parent_head_and_hash)); + pvd->get(), + maybe_parent_head && maybe_parent_head_hash + ? std::make_pair(std::cref(*maybe_parent_head), + std::cref(*maybe_parent_head_hash)) + : std::optional, + std::reference_wrapper>>{})); collations.status = CollationStatus::WaitingOnValidation; validateAsync( pending_collation_fetch.candidate_receipt, std::move(pending_collation_fetch.pov), - std::move(pvd), + std::move(pvd->get()), relay_parent); our_current_state_.validator_side.fetched_candidates.emplace( fetched_collation, collation_event); - return outcome::success(); + return outcome::success(true); } ParachainProcessorImpl::SecondingAllowed @@ -5496,14 +3211,12 @@ namespace kagome::parachain { "\n\t-> per_leaf={}" "\n\t-> per_candidate={}" "\n\t-> active_leaves={}" - "\n\t-> blocked_advertisements={}" "\n\t-> collation_requests_cancel_handles={}" "\n\t-> validator_side.fetched_candidates={}", our_current_state_.state_by_relay_parent.size(), our_current_state_.per_leaf.size(), our_current_state_.per_candidate.size(), - our_current_state_.active_leaves.size(), - our_current_state_.blocked_advertisements.size(), + our_current_state_.validator_side.active_leaves.size(), our_current_state_.collation_requests_cancel_handles.size(), our_current_state_.validator_side.fetched_candidates.size()); if (our_current_state_.implicit_view) { @@ -5515,12 +3228,12 @@ namespace kagome::parachain { av_store_->printStoragesLoad(); } - void ParachainProcessorImpl::handleAdvertisement( + void ParachainProcessorImpl::handle_advertisement( const RelayHash &relay_parent, const libp2p::peer::PeerId &peer_id, std::optional> &&prospective_candidate) { REINVOKE(*main_pool_handler_, - handleAdvertisement, + handle_advertisement, relay_parent, peer_id, std::move(prospective_candidate)); @@ -5589,15 +3302,6 @@ namespace kagome::parachain { ch, relay_parent, para_id); - - our_current_state_ - .blocked_advertisements[collator_para_id][parent_head_data_hash] - .emplace_back( - BlockedAdvertisement{.peer_id = peer_id, - .collator_id = collator_id, - .candidate_relay_parent = relay_parent, - .candidate_hash = ch}); - return; } } @@ -5819,14 +3523,4 @@ namespace kagome::parachain { return outcome::success(); } - bool ParachainProcessorImpl::canDisconnect(const libp2p::PeerId &peer) const { - auto audi = query_audi_->get(peer); - if (not audi) { - return true; - } - auto &peers = *peer_use_count_; - return SAFE_SHARED(peers) { - return peers.contains(*audi); - }; - } } // namespace kagome::parachain diff --git a/core/parachain/validator/network_bridge.hpp b/core/parachain/validator/network_bridge.hpp new file mode 100644 index 0000000000..6105365651 --- /dev/null +++ b/core/parachain/validator/network_bridge.hpp @@ -0,0 +1,130 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include "common/main_thread_pool.hpp" +#include "network/impl/stream_engine.hpp" +#include "network/peer_manager.hpp" +#include "network/protocol_base.hpp" +#include "utils/weak_macro.hpp" + +namespace kagome::parachain { + + struct NetworkBridge : std::enable_shared_from_this { + NetworkBridge( + common::MainThreadPool &main_thread_pool, + std::shared_ptr peer_manager, + std::shared_ptr app_state_manager) + : main_pool_handler_(main_thread_pool.handler(*app_state_manager)), + pm_(std::move(peer_manager)) {} + + template + void send_to_peer(const libp2p::peer::PeerId &peer, + const std::shared_ptr &protocol, + const std::shared_ptr &message) { + REINVOKE(*main_pool_handler_, + send_to_peer, + peer, + protocol, + std::move(message)); + std::ignore = tryOpenOutgoingStream( + peer, protocol, [WEAK_SELF, peer, message, protocol]() { + WEAK_LOCK(self); + self->pm_->getStreamEngine()->send(peer, protocol, message); + }); + } + + template + void send_response(std::shared_ptr stream, + std::shared_ptr protocol, + std::shared_ptr response) { + REINVOKE(*main_pool_handler_, + send_response, + std::move(stream), + std::move(protocol), + std::move(response)); + protocol->writeResponseAsync(std::move(stream), std::move(*response)); + } + + template + requires std::is_same_v + void connect_to_peers(Container peers) { + REINVOKE(*main_pool_handler_, connect_to_peers, std::move(peers)); + for (const auto &peer : peers) { + pm_->connectToPeer(libp2p::peer::PeerInfo{.id = peer}); + } + } + + template + requires std::is_same_v + void send_to_peers(Container peers, + const std::shared_ptr &protocol, + const std::shared_ptr &message) { + REINVOKE(*main_pool_handler_, + send_to_peers, + std::move(peers), + protocol, + message); + for (const auto &peer : peers) { + send_to_peer(peer, protocol, message); + } + } + + private: + template + bool tryOpenOutgoingStream(const libp2p::peer::PeerId &peer_id, + std::shared_ptr protocol, + F &&callback) { + auto stream_engine = pm_->getStreamEngine(); + BOOST_ASSERT(stream_engine); + + if (stream_engine->reserveOutgoing(peer_id, protocol)) { + protocol->newOutgoingStream( + peer_id, + [callback = std::forward(callback), + protocol, + peer_id, + wptr{weak_from_this()}](auto &&stream_result) mutable { + auto self = wptr.lock(); + if (not self) { + return; + } + + auto stream_engine = self->pm_->getStreamEngine(); + stream_engine->dropReserveOutgoing(peer_id, protocol); + + if (!stream_result.has_value()) { + self->logger->trace("Unable to create stream {} with {}: {}", + protocol->protocolName(), + peer_id, + stream_result.error()); + return; + } + + auto stream = stream_result.value(); + stream_engine->addOutgoing(std::move(stream_result.value()), + protocol); + + std::forward(callback)(); + }); + return true; + } + std::forward(callback)(); + return false; + } + + private: + log::Logger logger = log::createLogger("NetworkBridge", "parachain"); + std::shared_ptr main_pool_handler_; + std::shared_ptr pm_; + }; + +} // namespace kagome::parachain diff --git a/core/parachain/validator/parachain_processor.hpp b/core/parachain/validator/parachain_processor.hpp index e765327562..03fdc5a9fe 100644 --- a/core/parachain/validator/parachain_processor.hpp +++ b/core/parachain/validator/parachain_processor.hpp @@ -22,7 +22,6 @@ #include "consensus/timeline/slots_util.hpp" #include "crypto/hasher.hpp" #include "metrics/metrics.hpp" -#include "network/can_disconnect.hpp" #include "network/impl/stream_engine.hpp" #include "network/peer_manager.hpp" #include "network/peer_view.hpp" @@ -33,16 +32,15 @@ #include "parachain/availability/bitfield/signer.hpp" #include "parachain/availability/store/store.hpp" #include "parachain/backing/cluster.hpp" -#include "parachain/backing/grid_tracker.hpp" #include "parachain/backing/store.hpp" #include "parachain/pvf/precheck.hpp" #include "parachain/pvf/pvf.hpp" #include "parachain/validator/backing_implicit_view.hpp" #include "parachain/validator/collations.hpp" -#include "parachain/validator/impl/candidates.hpp" -#include "parachain/validator/impl/statements_store.hpp" #include "parachain/validator/prospective_parachains/prospective_parachains.hpp" #include "parachain/validator/signer.hpp" +#include "parachain/validator/statement_distribution/statement_distribution.hpp" +#include "parachain/validator/statement_distribution/types.hpp" #include "primitives/common.hpp" #include "primitives/event_types.hpp" #include "utils/non_copyable.hpp" @@ -92,6 +90,31 @@ namespace kagome::dispute { class RuntimeInfo; } +namespace kagome::parachain { + struct BlockedCollationId { + /// Para id. + ParachainId para_id; + /// Hash of the parent head data. + Hash parent_head_data_hash; + + BlockedCollationId(ParachainId pid, const Hash &h) + : para_id(pid), parent_head_data_hash(h) {} + constexpr auto operator<=>(const BlockedCollationId &) const = default; + }; +} // namespace kagome::parachain + +template <> +struct std::hash { + size_t operator()(const kagome::parachain::BlockedCollationId &value) const { + using Hash = kagome::parachain::Hash; + using ParachainId = kagome::parachain::ParachainId; + + size_t result = std::hash()(value.para_id); + boost::hash_combine(result, std::hash()(value.parent_head_data_hash)); + return result; + } +}; + namespace kagome::parachain { struct BackedCandidatesSource { @@ -102,7 +125,6 @@ namespace kagome::parachain { struct ParachainProcessorImpl : BackedCandidatesSource, - network::CanDisconnect, std::enable_shared_from_this { enum class Error { RESPONSE_ALREADY_RECEIVED = 1, @@ -173,8 +195,9 @@ namespace kagome::parachain { std::shared_ptr prospective_parachains, std::shared_ptr block_tree, LazySPtr slots_util, - std::shared_ptr - babe_config_repo); + std::shared_ptr babe_config_repo, + std::shared_ptr + statement_distribution); ~ParachainProcessorImpl() = default; /** @@ -196,7 +219,7 @@ namespace kagome::parachain { * @param prospective_candidate An optional pair containing the hash of the * prospective candidate and the hash of the parent block. */ - void handleAdvertisement( + void handle_advertisement( const RelayHash &relay_parent, const libp2p::peer::PeerId &peer_id, std::optional> &&prospective_candidate); @@ -253,10 +276,6 @@ namespace kagome::parachain { outcome::result OnFetchChunkRequestObsolete(const network::FetchChunkRequest &request); - outcome::result - OnFetchAttestedCandidateRequest( - const network::vstaging::AttestedCandidateRequest &request, - const libp2p::peer::PeerId &peer_id); outcome::result get_block_number_under_construction( const RelayHash &relay_parent) const; bool bitfields_indicate_availability( @@ -275,9 +294,6 @@ namespace kagome::parachain { std::vector getBackedCandidates( const RelayHash &relay_parent) override; - // CanDisconnect - bool canDisconnect(const libp2p::PeerId &) const override; - /** * @brief Fetches the Proof of Validity (PoV) for a given candidate. * @@ -295,6 +311,22 @@ namespace kagome::parachain { return backing_store_; } + /** + * @brief Handles a statement related to a specific relay parent. + * + * This function is responsible for processing a signed statement associated + * with a specific relay parent. The statement is provided in the form of a + * SignedFullStatementWithPVD object, which includes the payload and + * signature. The relay parent is identified by its block hash. + * + * @param relay_parent The block hash of the relay parent associated with + * the statement. + * @param statement The signed statement to be processed, encapsulated in a + * SignedFullStatementWithPVD object. + */ + void handleStatement(const primitives::BlockHash &relay_parent, + const SignedFullStatementWithPVD &statement); + private: enum struct StatementType { kSeconded = 0, kValid }; using Commitments = std::shared_ptr; @@ -339,93 +371,12 @@ namespace kagome::parachain { validity_votes; }; - struct StatementWithPVDSeconded { - network::CommittedCandidateReceipt committed_receipt; - runtime::PersistedValidationData pvd; - }; - - struct StatementWithPVDValid { - CandidateHash candidate_hash; - }; - - using StatementWithPVD = - boost::variant; - - using SignedFullStatementWithPVD = IndexedAndSigned; - - /** - * @brief Converts a SignedFullStatementWithPVD to an IndexedAndSigned - * CompactStatement. - */ - IndexedAndSigned signed_to_compact( - const SignedFullStatementWithPVD &s) const { - const Hash h = candidateHashFrom(getPayload(s)); - return { - .payload = - { - .payload = visit_in_place( - getPayload(s), - [&](const StatementWithPVDSeconded &) - -> network::vstaging::CompactStatement { - return network::vstaging::SecondedCandidateHash{ - .hash = h, - }; - }, - [&](const StatementWithPVDValid &) - -> network::vstaging::CompactStatement { - return network::vstaging::ValidCandidateHash{ - .hash = h, - }; - }), - .ix = s.payload.ix, - }, - .signature = s.signature, - }; - } - - /// polkadot/node/network/statement-distribution/src/v2/mod.rs - struct ActiveValidatorState { - // The index of the validator. - ValidatorIndex index; - // our validator group - GroupIndex group; - // the assignment of our validator group, if any. - std::optional assignment; - // the 'direct-in-group' communication at this relay-parent. - ClusterTracker cluster_tracker; - }; - - struct LocalValidatorState { - grid::GridTracker grid_tracker; - std::optional active; - }; - - using PeerUseCount = SafeObject< - std::unordered_map>; - - struct PerSessionState { - PerSessionState(const PerSessionState &) = delete; - PerSessionState &operator=(const PerSessionState &) = delete; - - PerSessionState(PerSessionState &&) = default; - PerSessionState &operator=(PerSessionState &&) = delete; - + struct PerSessionState final { SessionIndex session; runtime::SessionInfo session_info; - Groups groups; - std::optional grid_view; - std::optional our_index; - std::optional our_group; - std::shared_ptr peers; - - PerSessionState(SessionIndex _session, - runtime::SessionInfo _session_info, - Groups &&_groups, - grid::Views &&_grid_view, - std::optional _our_index, - std::shared_ptr peers); - ~PerSessionState(); - void updatePeers(bool add) const; + + PerSessionState(SessionIndex s, runtime::SessionInfo si) + : session(s), session_info(std::move(si)) {} }; struct RelayParentState { @@ -433,44 +384,22 @@ namespace kagome::parachain { std::optional assigned_core; std::optional assigned_para; std::vector> validator_to_group; - std::shared_ptr::RefObj> - per_session_state; - - std::optional our_index; - std::optional our_group; Collations collations; TableContext table_context; - std::optional statement_store; std::vector availability_cores; runtime::GroupDescriptor group_rotation_info; uint32_t minimum_backing_votes; - std::unordered_map - authority_lookup; - std::optional local_validator; std::unordered_set awaiting_validation; std::unordered_set issued_statements; std::unordered_set peers_advertised; std::unordered_map fallbacks; std::unordered_set backed_hashes; - std::unordered_set disabled_validators; bool inject_core_index; - - bool is_disabled(ValidatorIndex validator_index) const { - return disabled_validators.contains(validator_index); - } - - scale::BitVec disabled_bitmask( - const std::span &group) const { - scale::BitVec v; - v.bits.resize(group.size()); - for (size_t ix = 0; ix < group.size(); ++ix) { - v.bits[ix] = is_disabled(group[ix]); - } - return v; - } + std::shared_ptr::RefObj> + per_session_state; }; /** @@ -485,14 +414,6 @@ namespace kagome::parachain { Hash relay_parent; }; - using ManifestSummary = parachain::grid::ManifestSummary; - - struct ManifestImportSuccess { - bool acknowledge; - ValidatorIndex sender_index; - }; - using ManifestImportSuccessOpt = std::optional; - /* * Validation. */ @@ -549,22 +470,6 @@ namespace kagome::parachain { void makeAvailable(const primitives::BlockHash &candidate_hash, ValidateAndSecondResult &&result); - /** - * @brief Handles a statement related to a specific relay parent. - * - * This function is responsible for processing a signed statement associated - * with a specific relay parent. The statement is provided in the form of a - * SignedFullStatementWithPVD object, which includes the payload and - * signature. The relay parent is identified by its block hash. - * - * @param relay_parent The block hash of the relay parent associated with - * the statement. - * @param statement The signed statement to be processed, encapsulated in a - * SignedFullStatementWithPVD object. - */ - void handleStatement(const primitives::BlockHash &relay_parent, - const SignedFullStatementWithPVD &statement); - /** * @brief Processes a bitfield distribution message. * @@ -578,12 +483,6 @@ namespace kagome::parachain { void process_legacy_statement( const libp2p::peer::PeerId &peer_id, const network::StatementDistributionMessage &msg); - outcome::result handle_grid_statement( - const RelayHash &relay_parent, - ParachainProcessorImpl::RelayParentState &per_relay_parent, - grid::GridTracker &grid_tracker, - const IndexedAndSigned &statement, - ValidatorIndex grid_sender_index); /** * @brief Processes a vstaging statement. @@ -604,128 +503,14 @@ namespace kagome::parachain { const libp2p::peer::PeerId &peer_id, const network::vstaging::StatementDistributionMessage &msg); - /// Checks whether a statement is allowed, whether the signature is - /// accurate, - /// and importing into the cluster tracker if successful. - /// - /// if successful, this returns a checked signed statement if it should be - /// imported or otherwise an error indicating a reputational fault. - outcome::result> - handle_cluster_statement( - const RelayHash &relay_parent, - ClusterTracker &cluster_tracker, - SessionIndex session, - const runtime::SessionInfo &session_info, - const network::vstaging::SignedCompactStatement &statement, - ValidatorIndex cluster_sender_index); - - /// Check a statement signature under this parent hash. - outcome::result< - std::reference_wrapper> - check_statement_signature( - SessionIndex session_index, - const std::vector &validators, - const RelayHash &relay_parent, - const network::vstaging::SignedCompactStatement &statement); - void handle_incoming_manifest( - const libp2p::peer::PeerId &peer_id, - const network::vstaging::BackedCandidateManifest &msg); - void handle_incoming_statement( - const libp2p::peer::PeerId &peer_id, - const network::vstaging::StatementDistributionMessageStatement &stm); - void handle_incoming_acknowledgement( - const libp2p::peer::PeerId &peer_id, - const network::vstaging::BackedCandidateAcknowledgement - &acknowledgement); - void send_backing_fresh_statements( - const ConfirmedCandidate &confirmed, - const RelayHash &relay_parent, - ParachainProcessorImpl::RelayParentState &per_relay_parent, - const std::vector &group, - const CandidateHash &candidate_hash); - void apply_post_confirmation(const PostConfirmation &post_confirmation); - std::optional group_for_para( - const std::vector &availability_cores, - const runtime::GroupDescriptor &group_rotation_info, - ParachainId para_id) const; - void send_cluster_candidate_statements(const CandidateHash &candidate_hash, - const RelayHash &relay_parent); - void new_confirmed_candidate_fragment_chain_updates( - const HypotheticalCandidate &candidate); - void new_leaf_fragment_chain_updates(const Hash &leaf_hash); - void prospective_backed_notification_fragment_chain_updates( - ParachainId para_id, const Hash ¶_head); - void fragment_chain_update_inner( - std::optional> active_leaf_hash, - std::optional, - ParachainId>> required_parent_info, - std::optional> - known_hypotheticals); - void handleFetchedStatementResponse( - outcome::result &&r, - const RelayHash &relay_parent, - const CandidateHash &candidate_hash, - GroupIndex group_index); outcome::result getBabeRandomness( const RelayHash &relay_parent); outcome::result> fetch_claim_queue(const RelayHash &relay_parent); - void request_attested_candidate(const libp2p::peer::PeerId &peer, - RelayParentState &relay_parent_state, - const RelayHash &relay_parent, - const CandidateHash &candidate_hash, - GroupIndex group_index); - ManifestImportSuccessOpt handle_incoming_manifest_common( - const libp2p::peer::PeerId &peer_id, - const CandidateHash &candidate_hash, - const RelayHash &relay_parent, - ManifestSummary manifest_summary, - ParachainId para_id, - grid::ManifestKind manifest_kind); - network::vstaging::StatementFilter local_knowledge_filter( - size_t group_size, - GroupIndex group_index, - const CandidateHash &candidate_hash, - const StatementStore &statement_store); - std::deque, - network::VersionedValidatorProtocolMessage>> - acknowledgement_and_statement_messages( - const libp2p::peer::PeerId &peer, - network::CollationVersion version, - ValidatorIndex validator_index, - const Groups &groups, - ParachainProcessorImpl::RelayParentState &relay_parent_state, - const RelayHash &relay_parent, - GroupIndex group_index, - const CandidateHash &candidate_hash, - const network::vstaging::StatementFilter &local_knowledge); - std::deque - post_acknowledgement_statement_messages( - ValidatorIndex recipient, - const RelayHash &relay_parent, - grid::GridTracker &grid_tracker, - const StatementStore &statement_store, - const Groups &groups, - GroupIndex group_index, - const CandidateHash &candidate_hash, - const libp2p::peer::PeerId &peer, - network::CollationVersion version); void send_to_validators_group( const RelayHash &relay_parent, const std::deque &messages); - /** - * @brief Circulates a statement to the validators group. - * @param relay_parent The hash of the relay parent block. This is used to - * identify the group of validators to which the statement should be sent. - * @param statement The statement to be circulated. This is an indexed and - * signed compact statement. - */ - void circulate_statement( - const RelayHash &relay_parent, - RelayParentState &relay_parent_state, - const IndexedAndSigned &statement); - /** * @brief Inserts an advertisement into the peer's data. * @@ -809,7 +594,9 @@ namespace kagome::parachain { 32, crypto::Blake2b_StreamHasher<32>> &persisted_validation_data, - std::optional> maybe_parent_head_and_hash); + std::optional, + std::reference_wrapper>> + maybe_parent_head_and_hash); outcome::result> fetchPersistedValidationData(const RelayHash &relay_parent, @@ -819,6 +606,12 @@ namespace kagome::parachain { void onAttestNoPoVComplete(const network::RelayHash &relay_parent, const CandidateHash &candidate_hash); + /// Try seconding any collations which were waiting on the validation of + /// their parent + void second_unblocked_collations(ParachainId para_id, + const HeadData &head_data, + const Hash &head_data_hash); + void kickOffValidationWork( const RelayHash &relay_parent, AttestingData &attesting_data, @@ -831,8 +624,7 @@ namespace kagome::parachain { std::optional createAndSignStatement( const ValidateAndSecondResult &validation_result); template - outcome::result< - std::optional> + outcome::result> sign_import_and_distribute_statement( ParachainProcessorImpl::RelayParentState &rp_state, const ValidateAndSecondResult &validation_result); @@ -842,9 +634,7 @@ namespace kagome::parachain { std::optional &summary); template std::optional createAndSignStatementFromPayload( - T &&payload, - ValidatorIndex validator_ix, - RelayParentState ¶chain_state); + T &&payload, RelayParentState ¶chain_state); outcome::result> importStatement( const network::RelayHash &relay_parent, const SignedFullStatementWithPVD &statement, @@ -885,18 +675,6 @@ namespace kagome::parachain { return descriptor.collator_id; } - primitives::BlockHash candidateHashFrom( - const StatementWithPVD &statement) const { - return visit_in_place( - statement, - [&](const StatementWithPVDSeconded &val) { - return hasher_->blake2b_256( - ::scale::encode(val.committed_receipt.to_plain(*hasher_)) - .value()); - }, - [&](const StatementWithPVDValid &val) { return val.candidate_hash; }); - } - /* * Notification */ @@ -910,20 +688,11 @@ namespace kagome::parachain { BOOST_ASSERT(context); boost::asio::post(*context, std::forward(func)); } - void statementDistributionBackedCandidate( - const CandidateHash &candidate_hash); void notifyAvailableData(std::vector &&chunk_list, const primitives::BlockHash &relay_parent, const network::CandidateHash &candidate_hash, const network::ParachainBlock &pov, const runtime::PersistedValidationData &data); - void share_local_statement_vstaging( - RelayParentState &per_relay_parent, - const primitives::BlockHash &relay_parent, - const SignedFullStatementWithPVD &statement); - void share_local_statement_v1(RelayParentState &per_relay_parent, - const primitives::BlockHash &relay_parent, - const SignedFullStatementWithPVD &statement); template void notifySeconded(const primitives::BlockHash &relay_parent, const SignedFullStatementWithPVD &statement); @@ -940,11 +709,11 @@ namespace kagome::parachain { void onDeactivateBlocks( const primitives::events::RemoveAfterFinalizationParams &event); + void handle_active_leaves_update_for_validator(const network::ExView &event, + std::vector pruned); void onViewUpdated(const network::ExView &event); void OnBroadcastBitfields(const primitives::BlockHash &relay_parent, const network::SignedBitfield &bitfield); - void onUpdatePeerView(const libp2p::peer::PeerId &peer_id, - const network::View &view); outcome::result fetchCollation(const PendingCollation &pc, const CollatorId &id); outcome::result fetchCollation(const PendingCollation &pc, @@ -966,48 +735,12 @@ namespace kagome::parachain { RelayParentState &storeStateByRelayParent( const primitives::BlockHash &relay_parent, RelayParentState &&val); - /** - * @brief Sends peer messages corresponding for a given relay parent. - * - * @param peer_id Optional reference to the PeerId of the peer to send the - * messages to. - * @param relay_parent The hash of the relay parent block - */ - void send_peer_messages_for_relay_parent( - const libp2p::peer::PeerId &peer_id, const RelayHash &relay_parent); - std::optional, - network::VersionedValidatorProtocolMessage>> - pending_statement_network_message( - const StatementStore &statement_store, - const RelayHash &relay_parent, - const libp2p::peer::PeerId &peer, - network::CollationVersion version, - ValidatorIndex originator, - const network::vstaging::CompactStatement &compact); void prune_old_advertisements( const parachain::ImplicitView &implicit_view, const std::unordered_map &active_leaves, const std::unordered_map &per_relay_parent); - void provide_candidate_to_grid( - const CandidateHash &candidate_hash, - RelayParentState &relay_parent_state, - const ConfirmedCandidate &confirmed_candidate, - const runtime::SessionInfo &session_info); - void send_pending_cluster_statements( - const RelayHash &relay_parent, - const libp2p::peer::PeerId &peer_id, - network::CollationVersion version, - ValidatorIndex peer_validator_id, - ParachainProcessorImpl::RelayParentState &relay_parent_state); - void send_pending_grid_messages( - const RelayHash &relay_parent, - const libp2p::peer::PeerId &peer_id, - network::CollationVersion version, - ValidatorIndex peer_validator_id, - const Groups &groups, - ParachainProcessorImpl::RelayParentState &relay_parent_state); /** * The `create_backing_task` function is responsible for creating a new @@ -1022,7 +755,7 @@ namespace kagome::parachain { * @param relay_parent The hash of the relay parent block for which the * backing task is to be created. */ - void create_backing_task( + std::vector create_backing_task( const primitives::BlockHash &relay_parent, const network::HashedBlockHeader &block_header, const std::vector &deactivated); @@ -1043,16 +776,6 @@ namespace kagome::parachain { std::unordered_set &cache, const primitives::AuthorityDiscoveryId &id); - std::optional - find_active_validator_state( - ValidatorIndex validator_index, - const Groups &groups, - const std::vector &availability_cores, - const runtime::GroupDescriptor &group_rotation_info, - const std::optional &maybe_claim_queue, - size_t seconding_limit, - size_t max_candidate_depth); - template bool tryOpenOutgoingCollatingStream(const libp2p::peer::PeerId &peer_id, @@ -1123,16 +846,6 @@ namespace kagome::parachain { const std::shared_ptr &protocol); bool isValidatingNode() const; - void unblockAdvertisements( - ParachainProcessorImpl::RelayParentState &rp_state, - ParachainId para_id, - const Hash ¶_head); - void requestUnblockedCollations( - std::unordered_map< - ParachainId, - std::unordered_map>> - &&unblocked); - bool canSecond(ParachainId candidate_para_id, const Hash &candidate_relay_parent, const CandidateHash &candidate_hash, @@ -1145,7 +858,8 @@ namespace kagome::parachain { /// Handle a fetched collation result. /// polkadot/node/network/collator-protocol/src/validator_side/mod.rs - outcome::result kick_off_seconding( + /// Returns `true` if seconding started. + outcome::result kick_off_seconding( network::PendingCollationFetch &&pending_collation_fetch); std::optional importStatementToTable( @@ -1168,21 +882,18 @@ namespace kagome::parachain { std::optional implicit_view; std::unordered_map per_leaf; std::unordered_map per_candidate; - /// Added as independent member to prevent extra locks for - /// `state_by_relay_parent` which is used in internal thread only - std::unordered_map active_leaves; - std::unordered_map< - ParachainId, - std::unordered_map>> - blocked_advertisements; std::unordered_set collation_requests_cancel_handles; struct { + std::unordered_map active_leaves; std::unordered_map fetched_candidates; + std::unordered_map> + blocked_from_seconding; } validator_side; } our_current_state_; @@ -1190,7 +901,6 @@ namespace kagome::parachain { std::shared_ptr hasher_; std::shared_ptr peer_view_; network::PeerView::MyViewSubscriberPtr my_view_sub_; - network::PeerView::PeerViewSubscriberPtr remote_view_sub_; std::shared_ptr pvf_; std::shared_ptr signer_factory_; @@ -1204,8 +914,6 @@ namespace kagome::parachain { primitives::events::SyncStateSubscriptionEnginePtr sync_state_observable_; primitives::events::SyncStateEventSubscriberPtr sync_state_observer_; std::shared_ptr query_audi_; - std::shared_ptr> per_session_; - std::shared_ptr peer_use_count_; LazySPtr slots_util_; std::shared_ptr babe_config_repo_; @@ -1213,8 +921,10 @@ namespace kagome::parachain { std::shared_ptr worker_pool_handler_; std::default_random_engine random_; std::shared_ptr prospective_parachains_; - Candidates candidates_; std::shared_ptr block_tree_; + std::shared_ptr + statement_distribution; + std::shared_ptr> per_session; metrics::RegistryPtr metrics_registry_ = metrics::createRegistry(); metrics::Gauge *metric_is_parachain_validator_; diff --git a/core/parachain/validator/prospective_parachains/fragment_chain_errors.cpp b/core/parachain/validator/prospective_parachains/fragment_chain_errors.cpp index 3dd089f6e5..8dfda4c06c 100644 --- a/core/parachain/validator/prospective_parachains/fragment_chain_errors.cpp +++ b/core/parachain/validator/prospective_parachains/fragment_chain_errors.cpp @@ -7,8 +7,7 @@ #include "parachain/validator/prospective_parachains/fragment_chain_errors.hpp" #include "utils/stringify.hpp" -#define COMPONENT FragmentChain -#define COMPONENT_NAME STRINGIFY(COMPONENT) +#define COMPONENT_NAME "FragmentChain" OUTCOME_CPP_DEFINE_CATEGORY(kagome::parachain::fragment, FragmentChainError, diff --git a/core/parachain/validator/statement_distribution/peer_state.hpp b/core/parachain/validator/statement_distribution/peer_state.hpp new file mode 100644 index 0000000000..295b45d425 --- /dev/null +++ b/core/parachain/validator/statement_distribution/peer_state.hpp @@ -0,0 +1,89 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +#include "network/types/collator_messages_vstaging.hpp" +#include "parachain/validator/backing_implicit_view.hpp" +#include "utils/pool_handler_ready_make.hpp" + +namespace kagome::parachain { + struct ParachainProcessorImpl; +} + +namespace kagome::parachain::statement_distribution { + + struct PeerState { + network::View view; + std::unordered_set implicit_view; + + /// Update the view, returning a vector of implicit relay-parents which + /// weren't previously part of the view. + std::vector update_view( + const network::View &new_view, + const parachain::ImplicitView &local_implicit) { + std::unordered_set next_implicit; + for (const auto &x : new_view.heads_) { + auto t = + local_implicit.known_allowed_relay_parents_under(x, std::nullopt); + if (t) { + next_implicit.insert(t->begin(), t->end()); + } + } + + std::vector fresh_implicit; + for (const auto &x : next_implicit) { + if (implicit_view.find(x) == implicit_view.end()) { + fresh_implicit.emplace_back(x); + } + } + + view = new_view; + implicit_view = next_implicit; + return fresh_implicit; + } + + /// Whether we know that a peer knows a relay-parent. The peer knows the + /// relay-parent if it is either implicit or explicit in their view. + /// However, if it is implicit via an active-leaf we don't recognize, we + /// will not accurately be able to recognize them as 'knowing' the + /// relay-parent. + bool knows_relay_parent(const common::Hash256 &relay_parent) { + return implicit_view.contains(relay_parent) + || view.contains(relay_parent); + } + + /// Attempt to reconcile the view with new information about the implicit + /// relay parents under an active leaf. + std::vector reconcile_active_leaf( + const common::Hash256 &leaf_hash, + std::span implicit) { + if (!view.contains(leaf_hash)) { + return {}; + } + + std::vector v; + v.reserve(implicit.size()); + for (const auto &i : implicit) { + auto [_, inserted] = implicit_view.insert(i); + if (inserted) { + v.emplace_back(i); + } + } + return v; + } + }; + +} // namespace kagome::parachain::statement_distribution diff --git a/core/parachain/validator/statement_distribution/per_relay_parent_state.hpp b/core/parachain/validator/statement_distribution/per_relay_parent_state.hpp new file mode 100644 index 0000000000..cfe90c8480 --- /dev/null +++ b/core/parachain/validator/statement_distribution/per_relay_parent_state.hpp @@ -0,0 +1,76 @@ + +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +#include "common/ref_cache.hpp" +#include "parachain/backing/cluster.hpp" +#include "parachain/backing/grid_tracker.hpp" +#include "parachain/types.hpp" +#include "parachain/validator/impl/statements_store.hpp" +#include "parachain/validator/statement_distribution/per_session_state.hpp" +#include "utils/safe_object.hpp" + +namespace kagome::parachain::statement_distribution { + + /// polkadot/node/network/statement-distribution/src/v2/mod.rs + struct ActiveValidatorState { + // The index of the validator. + ValidatorIndex index; + // our validator group + GroupIndex group; + // the assignment of our validator group, if any. + std::optional assignment; + // the 'direct-in-group' communication at this relay-parent. + ClusterTracker cluster_tracker; + }; + + struct LocalValidatorState { + // the grid-level communication at this relay-parent. + grid::GridTracker grid_tracker; + // additional fields in case local node is an active validator. + std::optional active; + }; + + struct PerRelayParentState { + std::optional local_validator; + StatementStore statement_store; + size_t seconding_limit; + SessionIndex session; + std::unordered_map> groups_per_para; + std::unordered_set disabled_validators; + std::shared_ptr::RefObj> + per_session_state; + + std::optional> + active_validator_state() { + if (local_validator && local_validator->active) { + return local_validator->active.value(); + } + return std::nullopt; + } + + bool is_disabled(ValidatorIndex validator_index) const { + return disabled_validators.contains(validator_index); + } + + scale::BitVec disabled_bitmask( + const std::span &group) const { + scale::BitVec v; + v.bits.resize(group.size()); + for (size_t ix = 0; ix < group.size(); ++ix) { + v.bits[ix] = is_disabled(group[ix]); + } + return v; + } + }; + +} // namespace kagome::parachain::statement_distribution diff --git a/core/parachain/validator/statement_distribution/per_session_state.hpp b/core/parachain/validator/statement_distribution/per_session_state.hpp new file mode 100644 index 0000000000..80e15952f8 --- /dev/null +++ b/core/parachain/validator/statement_distribution/per_session_state.hpp @@ -0,0 +1,108 @@ + +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include +#include "parachain/backing/grid.hpp" +#include "parachain/groups.hpp" +#include "parachain/types.hpp" +#include "runtime/runtime_api/parachain_host_types.hpp" +#include "utils/safe_object.hpp" + +namespace kagome::parachain::statement_distribution { + + using LocalValidatorIndex = std::optional; + using PeerUseCount = + SafeObject>; + + struct PerSessionState final { + PerSessionState(const PerSessionState &) = delete; + PerSessionState &operator=(const PerSessionState &) = delete; + + PerSessionState(PerSessionState &&) = default; + PerSessionState &operator=(PerSessionState &&) = delete; + + SessionIndex session; + runtime::SessionInfo session_info; + Groups groups; + std::optional grid_view; + LocalValidatorIndex local_validator; + std::optional v_index; + std::shared_ptr peers; + std::unordered_map + authority_lookup; + + PerSessionState(SessionIndex _session, + runtime::SessionInfo _session_info, + Groups _groups, + grid::Views _grid_view, + LocalValidatorIndex _local_validator, + std::optional _v_index, + std::shared_ptr _peers, + std::unordered_map _authority_lookup) + : session{_session}, + session_info{std::move(_session_info)}, + groups{std::move(_groups)}, + grid_view{std::move(_grid_view)}, + local_validator(_local_validator), + v_index(_v_index), + peers(std::move(_peers)), + authority_lookup(std::move(_authority_lookup)) { + updatePeers(true); + } + + ~PerSessionState() { + updatePeers(false); + } + + void updatePeers(bool add) const { + const auto our_group = groups.byValidatorIndex(*local_validator); + if (!local_validator || !our_group || !this->peers) { + return; + } + auto &peers = *this->peers; + SAFE_UNIQUE(peers) { + auto f = [&](ValidatorIndex i) { + auto &id = session_info.discovery_keys[i]; + auto it = peers.find(id); + if (add) { + if (it == peers.end()) { + it = peers.emplace(id, 0).first; + } + ++it->second; + } else { + if (it == peers.end()) { + throw std::logic_error{"inconsistent PeerUseCount"}; + } + --it->second; + if (it->second == 0) { + peers.erase(it); + } + } + }; + for (auto &i : session_info.validator_groups[*our_group]) { + f(i); + } + if (grid_view) { + auto &view = grid_view->at(*our_group); + for (auto &i : view.sending) { + f(i); + } + for (auto &i : view.receiving) { + f(i); + } + } + }; + } + }; + +} // namespace kagome::parachain::statement_distribution diff --git a/core/parachain/validator/statement_distribution/statement_distribution.cpp b/core/parachain/validator/statement_distribution/statement_distribution.cpp new file mode 100644 index 0000000000..150207a91c --- /dev/null +++ b/core/parachain/validator/statement_distribution/statement_distribution.cpp @@ -0,0 +1,2937 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "network/impl/protocols/fetch_attested_candidate.hpp" +#include "network/impl/protocols/parachain_protocols.hpp" +#include "parachain/validator/parachain_processor.hpp" + +#define COMPONENT_NAME "StatementDistribution" + +OUTCOME_CPP_DEFINE_CATEGORY(kagome::parachain::statement_distribution, + StatementDistribution::Error, + e) { + using E = + kagome::parachain::statement_distribution::StatementDistribution::Error; + switch (e) { + case E::RESPONSE_ALREADY_RECEIVED: + return COMPONENT_NAME ": Response already present"; + case E::REJECTED_BY_PROSPECTIVE_PARACHAINS: + return COMPONENT_NAME ": Rejected by prospective parachains"; + case E::COLLATION_NOT_FOUND: + return COMPONENT_NAME ": Collation not found"; + case E::UNDECLARED_COLLATOR: + return COMPONENT_NAME ": Undeclared collator"; + case E::KEY_NOT_PRESENT: + return COMPONENT_NAME ": Private key is not present"; + case E::VALIDATION_FAILED: + return COMPONENT_NAME ": Validate and make available failed"; + case E::VALIDATION_SKIPPED: + return COMPONENT_NAME ": Validate and make available skipped"; + case E::OUT_OF_VIEW: + return COMPONENT_NAME ": Out of view"; + case E::CORE_INDEX_UNAVAILABLE: + return COMPONENT_NAME ": Core index unavailable"; + case E::DUPLICATE: + return COMPONENT_NAME ": Duplicate"; + case E::NO_INSTANCE: + return COMPONENT_NAME ": No self instance"; + case E::NOT_A_VALIDATOR: + return COMPONENT_NAME ": Node is not a validator"; + case E::NOT_SYNCHRONIZED: + return COMPONENT_NAME ": Node not synchronized"; + case E::PEER_LIMIT_REACHED: + return COMPONENT_NAME ": Peer limit reached"; + case E::PROTOCOL_MISMATCH: + return COMPONENT_NAME ": Protocol mismatch"; + case E::NOT_CONFIRMED: + return COMPONENT_NAME ": Candidate not confirmed"; + case E::NO_STATE: + return COMPONENT_NAME ": No parachain state"; + case E::NO_SESSION_INFO: + return COMPONENT_NAME ": No session info"; + case E::OUT_OF_BOUND: + return COMPONENT_NAME ": Index out of bound"; + case E::INCORRECT_BITFIELD_SIZE: + return COMPONENT_NAME ": Incorrect bitfield size"; + case E::INCORRECT_SIGNATURE: + return COMPONENT_NAME ": Incorrect signature"; + case E::CLUSTER_TRACKER_ERROR: + return COMPONENT_NAME ": Cluster tracker error"; + case E::PERSISTED_VALIDATION_DATA_NOT_FOUND: + return COMPONENT_NAME ": Persisted validation data not found"; + case E::PERSISTED_VALIDATION_DATA_MISMATCH: + return COMPONENT_NAME ": Persisted validation data mismatch"; + case E::CANDIDATE_HASH_MISMATCH: + return COMPONENT_NAME ": Candidate hash mismatch"; + case E::PARENT_HEAD_DATA_MISMATCH: + return COMPONENT_NAME ": Parent head data mismatch"; + case E::NO_PEER: + return COMPONENT_NAME ": No peer"; + case E::ALREADY_REQUESTED: + return COMPONENT_NAME ": Already requested"; + case E::NOT_ADVERTISED: + return COMPONENT_NAME ": Not advertised"; + case E::WRONG_PARA: + return COMPONENT_NAME ": Wrong para id"; + case E::THRESHOLD_LIMIT_REACHED: + return COMPONENT_NAME ": Threshold reached"; + } + return COMPONENT_NAME ": Unknown error"; +} + +#ifndef TRY_GET_OR_RET +#define TRY_GET_OR_RET(name, op) \ + auto name = (op); \ + if (!name) { \ + return; \ + } +#endif // TRY_GET_OR_RET + +#ifndef CHECK_OR_RET +#define CHECK_OR_RET(op) \ + if (!(op)) { \ + return; \ + } +#endif // CHECK_OR_RET + +namespace kagome::parachain::statement_distribution { + + StatementDistribution::StatementDistribution( + std::shared_ptr sf, + std::shared_ptr app_state_manager, + StatementDistributionThreadPool &statements_distribution_thread_pool, + std::shared_ptr _prospective_parachains, + std::shared_ptr _parachain_host, + std::shared_ptr _block_tree, + std::shared_ptr _query_audi, + std::shared_ptr _network_bridge, + std::shared_ptr _router, + common::MainThreadPool &main_thread_pool, + std::shared_ptr _hasher, + std::shared_ptr _crypto_provider, + std::shared_ptr _peer_view, + LazySPtr _slots_util, + std::shared_ptr _babe_config_repo, + primitives::events::PeerSubscriptionEnginePtr _peer_events_engine) + : implicit_view(_prospective_parachains, + _parachain_host, + _block_tree, + std::nullopt), + per_session(RefCache::create()), + signer_factory(std::move(sf)), + peer_use_count( + std::make_shared()), + statements_distribution_thread_handler( + poolHandlerReadyMake(this, + app_state_manager, + statements_distribution_thread_pool, + logger)), + query_audi(std::move(_query_audi)), + network_bridge(std::move(_network_bridge)), + router(std::move(_router)), + main_pool_handler{main_thread_pool.handler(*app_state_manager)}, + hasher(std::move(_hasher)), + prospective_parachains(_prospective_parachains), + parachain_host(_parachain_host), + crypto_provider(std::move(_crypto_provider)), + peer_view(_peer_view), + block_tree(_block_tree), + slots_util(_slots_util), + babe_config_repo(std::move(_babe_config_repo)), + peer_state_sub( + std::make_shared( + std::move(_peer_events_engine), false)), + my_view_sub(std::make_shared( + _peer_view->getMyViewObservable(), false)), + remote_view_sub(std::make_shared( + _peer_view->getRemoteViewObservable(), false)) { + BOOST_ASSERT(per_session); + BOOST_ASSERT(signer_factory); + BOOST_ASSERT(peer_use_count); + BOOST_ASSERT(statements_distribution_thread_handler); + BOOST_ASSERT(query_audi); + BOOST_ASSERT(network_bridge); + BOOST_ASSERT(router); + BOOST_ASSERT(main_pool_handler); + BOOST_ASSERT(hasher); + BOOST_ASSERT(prospective_parachains); + BOOST_ASSERT(parachain_host); + BOOST_ASSERT(crypto_provider); + BOOST_ASSERT(peer_view); + BOOST_ASSERT(block_tree); + BOOST_ASSERT(babe_config_repo); + BOOST_ASSERT(peer_state_sub); + BOOST_ASSERT(my_view_sub); + BOOST_ASSERT(remote_view_sub); + } + + bool StatementDistribution::tryStart() { + SL_INFO(logger, "StatementDistribution subsystem started."); + + primitives::events::subscribe( + *remote_view_sub, + network::PeerView::EventType::kViewUpdated, + [wptr{weak_from_this()}](const libp2p::peer::PeerId &peer_id, + const network::View &view) { + TRY_GET_OR_RET(self, wptr.lock()); + self->handle_peer_view_update(peer_id, view); + }); + + peer_state_sub->setCallback( + [wptr{weak_from_this()}](subscription::SubscriptionSetId, + auto &, + const auto ev_key, + const libp2p::peer::PeerId &peer) { + TRY_GET_OR_RET(self, wptr.lock()); + switch (ev_key) { + case primitives::events::PeerEventType::kConnected: + return self->on_peer_connected(peer); + case primitives::events::PeerEventType::kDisconnected: + return self->on_peer_disconnected(peer); + default: + break; + } + }); + peer_state_sub->subscribe(peer_state_sub->generateSubscriptionSetId(), + primitives::events::PeerEventType::kConnected); + peer_state_sub->subscribe(peer_state_sub->generateSubscriptionSetId(), + primitives::events::PeerEventType::kDisconnected); + + primitives::events::subscribe( + *my_view_sub, + network::PeerView::EventType::kViewUpdated, + [wptr{weak_from_this()}](const network::ExView &event) { + TRY_GET_OR_RET(self, wptr.lock()); + if (auto result = self->handle_view_event(event); + result.has_error()) { + SL_ERROR(self->logger, + "Handle view event failed. (relay parent={}, error={})", + event.new_head.hash(), + result.error()); + } + }); + + return true; + } + + void StatementDistribution::on_peer_connected( + const libp2p::peer::PeerId &peer) { + REINVOKE(*statements_distribution_thread_handler, on_peer_connected, peer); + auto _ = peers[peer]; + } + + void StatementDistribution::on_peer_disconnected( + const libp2p::peer::PeerId &peer) { + REINVOKE( + *statements_distribution_thread_handler, on_peer_disconnected, peer); + peers.erase(peer); + } + + outcome::result> + StatementDistribution::is_parachain_validator( + const primitives::BlockHash &relay_parent) const { + BOOST_ASSERT(main_pool_handler->isInCurrentThread()); + return signer_factory->at(relay_parent); + } + + outcome::result StatementDistribution::handle_view_event( + const network::ExView &event) { + BOOST_ASSERT(main_pool_handler->isInCurrentThread()); + if (auto parachain_proc = parachain_processor.lock(); + parachain_proc && parachain_proc->canProcessParachains().has_error()) { + return outcome::success(); + } + + OUTCOME_TRY(new_relay_parents, + implicit_view.exclusiveAccess( + [&](auto &iv) -> outcome::result> { + OUTCOME_TRY(iv.activate_leaf(event.new_head.hash())); + return outcome::success(iv.all_allowed_relay_parents()); + })); + + std::vector new_contexts; + new_contexts.reserve(new_relay_parents.size()); + + for (const auto &new_relay_parent : new_relay_parents) { + std::optional v_index; + if (auto res = + signer_factory->getAuthorityValidatorIndex(new_relay_parent); + res.has_value()) { + v_index = res.value(); + } + + std::optional validator_index; + if (auto res = is_parachain_validator(new_relay_parent); + res.has_value()) { + validator_index = utils::map(res.value(), [](const auto &signer) { + return signer.validatorIndex(); + }); + } + + new_contexts.emplace_back(RelayParentContext{ + .relay_parent = new_relay_parent, + .validator_index = validator_index, + .v_index = v_index, + }); + } + + handle_active_leaves_update(event, std::move(new_contexts)); + return outcome::success(); + } + + void StatementDistribution::handle_active_leaves_update( + const network::ExView &event, + std::vector new_contexts) { + REINVOKE(*statements_distribution_thread_handler, + handle_active_leaves_update, + event, + std::move(new_contexts)); + SL_TRACE(logger, + "New leaf update. (relay_parent={}, height={})", + event.new_head.hash(), + event.new_head.number); + if (auto res = + handle_active_leaves_update_inner(event, std::move(new_contexts)); + res.has_error()) { + SL_ERROR( + logger, + "Handle active leaf update inner failed. (relay parent={}, error={})", + event.new_head.hash(), + res.error()); + } + if (auto res = handle_deactive_leaves_update_inner(event.lost); + res.has_error()) { + SL_ERROR(logger, + "Handle deactive leaf update inner failed. (relay parent={}, " + "error={})", + event.new_head.hash(), + res.error()); + } + if (auto res = update_our_view(event.new_head.hash(), event.view); + res.has_error()) { + SL_ERROR(logger, + "Update our view failed. (relay parent={}, error={})", + event.new_head.hash(), + res.error()); + } + } + + outcome::result + StatementDistribution::handle_active_leaves_update_inner( + const network::ExView &event, + std::vector new_contexts) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + const auto &relay_parent = event.new_head.hash(); + const auto leaf_mode = + prospective_parachains->prospectiveParachainsMode(relay_parent); + if (!leaf_mode) { + return outcome::success(); + } + + const auto max_candidate_depth = leaf_mode->max_candidate_depth; + const auto seconding_limit = max_candidate_depth + 1; + + for (const auto &new_context : new_contexts) { + const auto &new_relay_parent = new_context.relay_parent; + + /// We check `per_relay_state` befor `per_session_state`, because we keep + /// ref in `per_relay_parent` directly + if (tryGetStateByRelayParent(new_relay_parent)) { + continue; + } + + OUTCOME_TRY(session_index, + parachain_host->session_index_for_child(new_relay_parent)); + OUTCOME_TRY(disabled_validators_, + parachain_host->disabled_validators(new_relay_parent)); + std::unordered_set disabled_validators{ + disabled_validators_.begin(), disabled_validators_.end()}; + if (!disabled_validators.empty()) { + SL_TRACE(logger, + "Disabled validators detected. (relay parent={})", + new_relay_parent); + } + + auto per_session_state = per_session->get_or_insert( + session_index, + [&]() -> outcome::result< + RefCache::RefObj> { + OUTCOME_TRY( + session_info, + parachain_host->session_info(new_relay_parent, session_index)); + if (!new_context.v_index) { + SL_TRACE(logger, + "Not a validator. (new_relay_parent={})", + new_relay_parent); + return outcome::failure(Error::NOT_A_VALIDATOR); + } + + uint32_t minimum_backing_votes = 2; /// legacy value + if (auto r = parachain_host->minimum_backing_votes(new_relay_parent, + session_index); + r.has_value()) { + minimum_backing_votes = r.value(); + } else { + SL_TRACE(logger, + "Querying the backing threshold from the runtime is not " + "supported by the current Runtime API. " + "(new_relay_parent={})", + new_relay_parent); + } + + OUTCOME_TRY(block_header, + block_tree->getBlockHeader(new_relay_parent)); + OUTCOME_TRY(babe_header, + consensus::babe::getBabeBlockHeader(block_header)); + OUTCOME_TRY( + epoch, + slots_util.get()->slotToEpoch(*block_header.parentInfo(), + babe_header.slot_number)); + OUTCOME_TRY( + babe_config, + babe_config_repo->config(*block_header.parentInfo(), epoch)); + std::unordered_map + authority_lookup; + for (ValidatorIndex v = 0; v < session_info->discovery_keys.size(); + ++v) { + authority_lookup[session_info->discovery_keys[v]] = v; + } + + grid::Views grid_view = grid::makeViews( + session_info->validator_groups, + grid::shuffle(session_info->discovery_keys.size(), + babe_config->randomness), + *new_context.v_index); + + return outcome::success( + RefCache::RefObj( + session_index, + *session_info, + Groups{session_info->validator_groups, + minimum_backing_votes}, + std::move(grid_view), + new_context.validator_index, + new_context.v_index, + peer_use_count, + std::move(authority_lookup))); + }); + if (per_session_state.has_error()) { + SL_WARN(logger, + "Create session data failed. (error={})", + per_session_state.error()); + continue; + } + + OUTCOME_TRY(availability_cores, + parachain_host->availability_cores(new_relay_parent)); + OUTCOME_TRY(groups, parachain_host->validator_groups(new_relay_parent)); + const auto &[_, group_rotation_info] = groups; + + auto maybe_claim_queue = + [&]() -> std::optional { + auto r = fetch_claim_queue(new_relay_parent); + if (r.has_value()) { + return r.value(); + } + return std::nullopt; + }(); + + auto local_validator = [&]() -> std::optional { + if (!per_session_state.value()->value().v_index) { + return std::nullopt; + } + + if (per_session_state.value()->value().local_validator) { + return find_active_validator_state( + *per_session_state.value()->value().local_validator, + per_session_state.value()->value().groups, + availability_cores, + group_rotation_info, + maybe_claim_queue, + seconding_limit, + max_candidate_depth); + } + return LocalValidatorState{}; + }(); + + auto groups_per_para = determine_groups_per_para(availability_cores, + group_rotation_info, + maybe_claim_queue, + max_candidate_depth); + per_relay_parent.emplace( + new_relay_parent, + PerRelayParentState{ + .local_validator = local_validator, + .statement_store = + StatementStore(per_session_state.value()->value().groups), + .seconding_limit = seconding_limit, + .session = session_index, + .groups_per_para = std::move(groups_per_para), + .disabled_validators = std::move(disabled_validators), + .per_session_state = per_session_state.value(), + }); + } // for + + SL_TRACE( + logger, + "Activated leaves. Now tracking {} relay-parents across {} sessions", + per_relay_parent.size(), + per_session->size()); + + std::vector new_relay_parents; + new_relay_parents.reserve(new_contexts.size()); + std::ranges::transform( + new_contexts, new_relay_parents.begin(), [](const auto &context) { + return context.relay_parent; + }); + + std::vector>> + update_peers; + for (auto &[peer, peer_state] : peers) { + std::vector fresh = + peer_state.reconcile_active_leaf(relay_parent, new_relay_parents); + if (!fresh.empty()) { + update_peers.emplace_back(peer, fresh); + } + } + + for (const auto &[peer, fresh] : update_peers) { + for (const auto &fresh_relay_parent : fresh) { + send_peer_messages_for_relay_parent(peer, fresh_relay_parent); + } + } + + new_leaf_fragment_chain_updates(relay_parent); + return outcome::success(); + } + + outcome::result + StatementDistribution::handle_deactive_leaves_update_inner( + const std::vector &lost) { + implicit_view.exclusiveAccess([&](auto &iv) { + for (const auto &leaf : lost) { + const auto pruned = iv.deactivate_leaf(leaf); + for (const auto &pruned_rp : pruned) { + if (auto it = per_relay_parent.find(pruned_rp); + it != per_relay_parent.end()) { + if (auto active_state = it->second.active_validator_state()) { + active_state->get() + .cluster_tracker.warn_if_too_many_pending_statements( + pruned_rp); + } + per_relay_parent.erase(it); + } + } + } + }); + + candidates.on_deactivate_leaves( + lost, [&](const auto &h) { return per_relay_parent.contains(h); }); + return outcome::success(); + } + + outcome::result StatementDistribution::update_our_view( + const Hash &relay_parent, const network::View &view) { + if (auto parachain_proc = parachain_processor.lock()) { + OUTCOME_TRY(parachain_proc->canProcessParachains()); + } + + OUTCOME_TRY(per_relay_parent, getStateByRelayParent(relay_parent)); + + std::unordered_set peers_to_send; + const auto &per_session_state = + per_relay_parent.get().per_session_state->value(); + const auto &local_validator = per_session_state.local_validator; + + if (local_validator) { + if (const auto our_group = + per_session_state.groups.byValidatorIndex(*local_validator)) { + /// update peers of our group + if (const auto group = per_session_state.groups.get(*our_group)) { + for (const auto vi : *group) { + if (auto peer = query_audi->get( + per_session_state.session_info.discovery_keys[vi])) { + peers_to_send.emplace(peer->id); + } else { + SL_TRACE(logger, + "No audi for {}.", + per_session_state.session_info.discovery_keys[vi]); + } + } + } + } + } + + /// update peers in grid view + if (per_session_state.grid_view) { + for (const auto &view : *per_session_state.grid_view) { + for (const auto vi : view.sending) { + if (auto peer = query_audi->get( + per_session_state.session_info.discovery_keys[vi])) { + peers_to_send.emplace(peer->id); + } else { + SL_TRACE(logger, + "No audi for {}.", + per_session_state.session_info.discovery_keys[vi]); + } + } + for (const auto vi : view.receiving) { + if (auto peer = query_audi->get( + per_session_state.session_info.discovery_keys[vi])) { + peers_to_send.emplace(peer->id); + } else { + SL_TRACE(logger, + "No audi for {}.", + per_session_state.session_info.discovery_keys[vi]); + } + } + } + } + + for (const auto &[peer, _] : peers) { + peers_to_send.emplace(peer); + } + + SL_INFO(logger, "Send my view. (peers_count={})", peers_to_send.size()); + auto message = std::make_shared< + network::WireMessage>( + network::ViewUpdate{ + .view = view, + }); + network_bridge->connect_to_peers(peers_to_send); + network_bridge->send_to_peers( + peers_to_send, router->getValidationProtocolVStaging(), message); + return outcome::success(); + } + + std::unordered_map> + StatementDistribution::determine_groups_per_para( + const std::vector &availability_cores, + const runtime::GroupDescriptor &group_rotation_info, + std::optional &maybe_claim_queue, + size_t max_candidate_depth) { + const auto n_cores = availability_cores.size(); + const auto schedule = + [&]() -> std::unordered_map> { + if (maybe_claim_queue) { + return {maybe_claim_queue->claimes.begin(), + maybe_claim_queue->claimes.end()}; + } + + std::unordered_map> result; + for (CoreIndex index = 0; index < availability_cores.size(); ++index) { + const auto &core = availability_cores[index]; + visit_in_place( + core, + [&](const runtime::ScheduledCore &scheduled_core) { + result.emplace(index, + std::vector{scheduled_core.para_id}); + }, + [&](const runtime::OccupiedCore &occupied_core) { + if (max_candidate_depth >= 1) { + if (occupied_core.next_up_on_available) { + result.emplace( + index, + std::vector{ + occupied_core.next_up_on_available->para_id}); + } + } + }, + [&](const auto &) {}); + } + return result; + }(); + + std::unordered_map> groups_per_para; + for (const auto &[core_index, paras] : schedule) { + const auto group_index = + group_rotation_info.groupForCore(core_index, n_cores); + for (const auto ¶ : paras) { + groups_per_para[para].emplace_back(group_index); + } + } + return groups_per_para; + } + + outcome::result> + StatementDistribution::fetch_claim_queue(const RelayHash &relay_parent) { + // constexpr uint32_t CLAIM_QUEUE_RUNTIME_REQUIREMENT = 11; + // OUTCOME_TRY(version, + // parachain_host->runtime_api_version(relay_parent)); if (version < + // CLAIM_QUEUE_RUNTIME_REQUIREMENT) { + // SL_TRACE(logger, "Runtime doesn't support `request_claim_queue`"); + // return std::nullopt; + // } + + OUTCOME_TRY(claims, parachain_host->claim_queue(relay_parent)); + return runtime::ClaimQueueSnapshot{ + .claimes = std::move(claims), + }; + } + + bool StatementDistribution::can_disconnect(const libp2p::PeerId &peer) const { + auto audi = query_audi->get(peer); + if (not audi) { + return true; + } + auto &peers = *peer_use_count; + return SAFE_SHARED(peers) { + return peers.contains(*audi); + }; + } + + std::optional> + StatementDistribution::tryGetStateByRelayParent( + const primitives::BlockHash &relay_parent) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + const auto it = per_relay_parent.find(relay_parent); + if (it != per_relay_parent.end()) { + return it->second; + } + return std::nullopt; + } + + outcome::result> + StatementDistribution::getStateByRelayParent( + const primitives::BlockHash &relay_parent) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + if (auto per_relay_parent = tryGetStateByRelayParent(relay_parent)) { + return *per_relay_parent; + } + return Error::OUT_OF_VIEW; + } + + // outcome::result + void StatementDistribution::OnFetchAttestedCandidateRequest( + const network::vstaging::AttestedCandidateRequest &request, + std::shared_ptr stream) { + REINVOKE(*statements_distribution_thread_handler, + OnFetchAttestedCandidateRequest, + request, + stream); + + const auto peer_res = stream->remotePeerId(); + if (peer_res.has_error()) { + SL_ERROR(logger, + "Fetch attested candidate failed. No remote peer data. " + "(candidate hash={})", + request.candidate_hash); + return; + } + + const libp2p::peer::PeerId &peer_id = peer_res.value(); + auto confirmed = candidates.get_confirmed(request.candidate_hash); + if (!confirmed) { + SL_ERROR( + logger, + "Fetch attested candidate failed. Not confirmed. (candidate hash={})", + request.candidate_hash); + return; + } + + auto relay_parent_state = + tryGetStateByRelayParent(confirmed->get().relay_parent()); + if (!relay_parent_state) { + SL_ERROR(logger, + "Fetch attested candidate failed. Out of view. (candidate " + "hash={}, relay parent={})", + request.candidate_hash, + confirmed->get().relay_parent()); + return; + } + + auto &local_validator = relay_parent_state->get().local_validator; + if (!local_validator) { + SL_ERROR(logger, + "Fetch attested candidate failed. Not a validator. (candidate " + "hash={}, relay parent={})", + request.candidate_hash, + confirmed->get().relay_parent()); + return; + } + + const auto &session_info = + relay_parent_state->get().per_session_state->value().session_info; + const auto &groups = + relay_parent_state->get().per_session_state->value().groups; + auto group = groups.get(confirmed->get().group_index()); + if (!group) { + SL_ERROR(logger, + "Unexpected array bound for groups. (candidate hash={}, relay " + "parent={})", + request.candidate_hash, + confirmed->get().relay_parent()); + return; + } + + const auto group_size = group->size(); + auto &mask = request.mask; + if (mask.seconded_in_group.bits.size() != group_size + || mask.validated_in_group.bits.size() != group_size) { + SL_ERROR(logger, + "Fetch attested candidate failed. Incorrect bitfield size. " + "(candidate hash={}, relay parent={})", + request.candidate_hash, + confirmed->get().relay_parent()); + return; + } + + auto &active = local_validator->active; + std::optional validator_id; + bool is_cluster = false; + [&] { + auto audi = query_audi->get(peer_id); + if (not audi.has_value()) { + SL_TRACE(logger, "No audi. (peer={})", peer_id); + return; + } + + ValidatorIndex v = 0; + for (; v < session_info.discovery_keys.size(); ++v) { + if (session_info.discovery_keys[v] == audi.value()) { + SL_TRACE(logger, + "Captured validator. (relay_parent={}, candidate_hash={})", + confirmed->get().relay_parent(), + request.candidate_hash); + break; + } + } + + if (v >= session_info.discovery_keys.size()) { + return; + } + + if (active + and active->cluster_tracker.can_request(v, request.candidate_hash)) { + validator_id = v; + is_cluster = true; + + } else if (local_validator->grid_tracker.can_request( + v, request.candidate_hash)) { + validator_id = v; + } + }(); + + if (!validator_id) { + SL_ERROR(logger, + "Fetch attested candidate failed. Out of bound. (candidate " + "hash={}, relay parent={})", + request.candidate_hash, + confirmed->get().relay_parent()); + return; + } + + auto init_with_not = [](scale::BitVec &dst, const scale::BitVec &src) { + dst.bits.reserve(src.bits.size()); + for (const auto i : src.bits) { + dst.bits.emplace_back(!i); + } + }; + + network::vstaging::StatementFilter and_mask; + init_with_not(and_mask.seconded_in_group, request.mask.seconded_in_group); + init_with_not(and_mask.validated_in_group, request.mask.validated_in_group); + + std::vector> + statements; + network::vstaging::StatementFilter sent_filter(group_size); + relay_parent_state->get().statement_store.groupStatements( + *group, + request.candidate_hash, + and_mask, + [&](const IndexedAndSigned + &statement) { + for (size_t ix = 0; ix < group->size(); ++ix) { + if ((*group)[ix] == statement.payload.ix) { + visit_in_place( + getPayload(statement).inner_value, + [&](const network::vstaging::SecondedCandidateHash &) { + sent_filter.seconded_in_group.bits[ix] = true; + }, + [&](const network::vstaging::ValidCandidateHash &) { + sent_filter.validated_in_group.bits[ix] = true; + }, + [&](const auto &) {}); + } + } + statements.emplace_back(statement); + }); + + if (!is_cluster) { + auto threshold = std::get<1>(*groups.get_size_and_backing_threshold( + confirmed->get().group_index())); + const auto seconded_and_sufficient = + (sent_filter.has_seconded() + && sent_filter.backing_validators() >= threshold); + if (!seconded_and_sufficient) { + SL_INFO(logger, + "Dropping a request from a grid peer because the backing " + "threshold is no longer met. (relay_parent={}, " + "candidate_hash={}, group_index={})", + confirmed->get().relay_parent(), + request.candidate_hash, + confirmed->get().group_index()); + return; + } + } + + for (const auto &statement : statements) { + if (is_cluster) { + active->cluster_tracker.note_sent( + *validator_id, + statement.payload.ix, + network::vstaging::from(getPayload(statement))); + } else { + local_validator->grid_tracker.sent_or_received_direct_statement( + groups, + statement.payload.ix, + *validator_id, + getPayload(statement), + false); + } + } + + network_bridge->send_response( + stream, + router->getFetchAttestedCandidateProtocol(), + std::make_shared( + network::vstaging::AttestedCandidateResponse{ + .candidate_receipt = confirmed->get().receipt, + .persisted_validation_data = + confirmed->get().persisted_validation_data, + .statements = std::move(statements), + })); + } + + void StatementDistribution::handle_peer_view_update( + const libp2p::peer::PeerId &peer, const network::View &new_view) { + REINVOKE(*statements_distribution_thread_handler, + handle_peer_view_update, + peer, + new_view); + + auto &peer_state = peers[peer]; + implicit_view.sharedAccess([&](const auto &iv) { + auto fresh_implicit = peer_state.update_view(new_view, iv); + for (const auto &new_relay_parent : fresh_implicit) { + send_peer_messages_for_relay_parent(peer, new_relay_parent); + } + }); + } + + outcome::result StatementDistribution::handle_grid_statement( + const RelayHash &relay_parent, + PerRelayParentState &per_relay_parent, + grid::GridTracker &grid_tracker, + const IndexedAndSigned &statement, + ValidatorIndex grid_sender_index) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + /// TODO(iceseer): do Ensure the statement is correctly signed. Signature + /// check. + grid_tracker.sent_or_received_direct_statement( + per_relay_parent.per_session_state->value().groups, + statement.payload.ix, + grid_sender_index, + getPayload(statement), + true); + return outcome::success(); + } + + network::vstaging::StatementFilter + StatementDistribution::local_knowledge_filter( + size_t group_size, + GroupIndex group_index, + const CandidateHash &candidate_hash, + const StatementStore &statement_store) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + network::vstaging::StatementFilter f{group_size}; + statement_store.fill_statement_filter(group_index, candidate_hash, f); + return f; + } + + void StatementDistribution::request_attested_candidate( + const libp2p::peer::PeerId &peer, + PerRelayParentState &relay_parent_state, + const RelayHash &relay_parent, + const CandidateHash &candidate_hash, + GroupIndex group_index) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + CHECK_OR_RET(relay_parent_state.local_validator); + auto &local_validator = *relay_parent_state.local_validator; + + const auto &session_info = + relay_parent_state.per_session_state->value().session_info; + + TRY_GET_OR_RET( + group, + relay_parent_state.per_session_state->value().groups.get(group_index)); + const auto seconding_limit = relay_parent_state.seconding_limit; + + SL_TRACE(logger, + "Form unwanted mask. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + network::vstaging::StatementFilter unwanted_mask(group->size()); + for (size_t i = 0; i < group->size(); ++i) { + const auto v = (*group)[i]; + if (relay_parent_state.statement_store.seconded_count(v) + >= seconding_limit) { + unwanted_mask.seconded_in_group.bits[i] = true; + } + } + + auto disabled_mask = relay_parent_state.disabled_bitmask(*group); + if (disabled_mask.bits.size() + > unwanted_mask.seconded_in_group.bits.size()) { + unwanted_mask.seconded_in_group.bits.resize(disabled_mask.bits.size()); + } + if (disabled_mask.bits.size() + > unwanted_mask.validated_in_group.bits.size()) { + unwanted_mask.validated_in_group.bits.resize(disabled_mask.bits.size()); + } + for (size_t i = 0; i < disabled_mask.bits.size(); ++i) { + unwanted_mask.seconded_in_group.bits[i] = + unwanted_mask.seconded_in_group.bits[i] || disabled_mask.bits[i]; + unwanted_mask.validated_in_group.bits[i] = + unwanted_mask.validated_in_group.bits[i] || disabled_mask.bits[i]; + } + + auto backing_threshold = [&]() -> std::optional { + auto bt = relay_parent_state.per_session_state->value() + .groups.get_size_and_backing_threshold(group_index); + return bt ? std::get<1>(*bt) : std::optional{}; + }(); + + SL_TRACE(logger, + "Enumerate peers. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + std::optional target; + auto audi = query_audi->get(peer); + if (!audi) { + SL_TRACE(logger, + "No audi. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + return; + } + + ValidatorIndex validator_id = 0; + for (; validator_id < session_info.discovery_keys.size(); ++validator_id) { + if (session_info.discovery_keys[validator_id] == *audi) { + SL_TRACE(logger, + "Captured validator. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + break; + } + } + + CHECK_OR_RET(validator_id < session_info.discovery_keys.size()); + auto filter = [&]() -> std::optional { + if (local_validator.active) { + if (local_validator.active->cluster_tracker.knows_candidate( + validator_id, candidate_hash)) { + return network::vstaging::StatementFilter( + local_validator.active->cluster_tracker.targets().size()); + } + } + + auto filter = local_validator.grid_tracker.advertised_statements( + validator_id, candidate_hash); + if (filter) { + return filter; + } + + SL_TRACE(logger, + "No filter. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + return std::nullopt; + }(); + + CHECK_OR_RET(filter); + filter->mask_seconded(unwanted_mask.seconded_in_group); + filter->mask_valid(unwanted_mask.validated_in_group); + + if (!backing_threshold + || (filter->has_seconded() + && filter->backing_validators() >= *backing_threshold)) { + target.emplace(peer); + } else { + SL_TRACE( + logger, + "Not pass backing threshold. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + return; + } + + if (!target) { + SL_TRACE(logger, + "Target not found. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + return; + } + + SL_TRACE(logger, + "Requesting. (peer={}, relay_parent={}, candidate_hash={})", + peer, + relay_parent, + candidate_hash); + router->getFetchAttestedCandidateProtocol()->doRequest( + peer, + network::vstaging::AttestedCandidateRequest{ + .candidate_hash = candidate_hash, + .mask = unwanted_mask, + }, + [wptr{weak_from_this()}, + relay_parent{relay_parent}, + candidate_hash{candidate_hash}, + group_index{group_index}]( + outcome::result + r) mutable { + TRY_GET_OR_RET(self, wptr.lock()); + self->handle_response( + std::move(r), relay_parent, candidate_hash, group_index); + }); + } + + void StatementDistribution::handle_response( + outcome::result &&r, + const RelayHash &relay_parent, + const CandidateHash &candidate_hash, + GroupIndex group_index) { + REINVOKE(*statements_distribution_thread_handler, + handle_response, + std::move(r), + relay_parent, + candidate_hash, + group_index); + + if (r.has_error()) { + SL_INFO(logger, + "Fetch attested candidate returned an error. (relay parent={}, " + "candidate={}, group index={}, error={})", + relay_parent, + candidate_hash, + group_index, + r.error()); + return; + } + + TRY_GET_OR_RET(parachain_state, tryGetStateByRelayParent(relay_parent)); + TRY_GET_OR_RET(group, + parachain_state->get().per_session_state->value().groups.get( + group_index)); + + [[maybe_unused]] const auto disabled_mask = + parachain_state->get().disabled_bitmask(*group); + const network::vstaging::AttestedCandidateResponse &response = r.value(); + SL_INFO(logger, + "Fetch attested candidate success. (relay parent={}, " + "candidate={}, group index={}, statements={})", + relay_parent, + candidate_hash, + group_index, + response.statements.size()); + for (const auto &statement : response.statements) { + parachain_state->get().statement_store.insert( + parachain_state->get().per_session_state->value().groups, + statement, + StatementOrigin::Remote); + } + + auto opt_post_confirmation = + candidates.confirm_candidate(candidate_hash, + response.candidate_receipt, + response.persisted_validation_data, + group_index, + hasher); + if (!opt_post_confirmation) { + SL_WARN(logger, + "Candidate re-confirmed by request/response: logic error. (relay " + "parent={}, candidate={})", + relay_parent, + candidate_hash); + return; + } + + auto &post_confirmation = *opt_post_confirmation; + apply_post_confirmation(post_confirmation); + + auto opt_confirmed = candidates.get_confirmed(candidate_hash); + BOOST_ASSERT(opt_confirmed); + + if (!opt_confirmed->get().is_importable(std::nullopt)) { + SL_INFO(logger, + "Not importable. (relay parent={}, " + "candidate={}, group index={})", + relay_parent, + candidate_hash, + group_index); + return; + } + + const auto &groups = + parachain_state->get().per_session_state->value().groups; + auto it = groups.groups.find(group_index); + if (it == groups.groups.end()) { + SL_WARN(logger, + "Group was not found. (relay parent={}, candidate={}, group " + "index={})", + relay_parent, + candidate_hash, + group_index); + return; + } + + SL_INFO(logger, + "Send fresh statements. (relay parent={}, " + "candidate={})", + relay_parent, + candidate_hash); + send_backing_fresh_statements(opt_confirmed->get(), + relay_parent, + parachain_state->get(), + it->second, + candidate_hash); + } + + void StatementDistribution::apply_post_confirmation( + const PostConfirmation &post_confirmation) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + const auto candidate_hash = candidateHash(post_confirmation.hypothetical); + send_cluster_candidate_statements( + candidate_hash, relayParent(post_confirmation.hypothetical)); + + new_confirmed_candidate_fragment_chain_updates( + post_confirmation.hypothetical); + } + + void StatementDistribution::send_cluster_candidate_statements( + const CandidateHash &candidate_hash, const RelayHash &relay_parent) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + TRY_GET_OR_RET(relay_parent_state, tryGetStateByRelayParent(relay_parent)); + TRY_GET_OR_RET( + local_group, + utils::map(relay_parent_state->get().active_validator_state(), + [](const auto &state) { return state.get().group; })); + TRY_GET_OR_RET( + group, + relay_parent_state->get().per_session_state->value().groups.get( + *local_group)); + + auto group_size = group->size(); + relay_parent_state->get().statement_store.groupStatements( + *group, + candidate_hash, + network::vstaging::StatementFilter(group_size, true), + [&](const IndexedAndSigned + &statement) { + circulate_statement( + relay_parent, relay_parent_state->get(), statement); + }); + } + + void StatementDistribution::handle_backed_candidate_message( + const CandidateHash &candidate_hash) { + REINVOKE(*statements_distribution_thread_handler, + handle_backed_candidate_message, + candidate_hash); + + auto confirmed_opt = candidates.get_confirmed(candidate_hash); + if (!confirmed_opt) { + SL_TRACE(logger, + "Received backed candidate notification for unknown or " + "unconfirmed. (candidate_hash={})", + candidate_hash); + return; + } + const auto &confirmed = confirmed_opt->get(); + + const auto relay_parent = confirmed.relay_parent(); + TRY_GET_OR_RET(relay_parent_state_opt, + tryGetStateByRelayParent(relay_parent)); + + const auto &session_info = + relay_parent_state_opt->get().per_session_state->value().session_info; + provide_candidate_to_grid( + candidate_hash, relay_parent_state_opt->get(), confirmed, session_info); + + prospective_backed_notification_fragment_chain_updates( + confirmed.para_id(), confirmed.para_head()); + } + + void StatementDistribution::handle_incoming_acknowledgement( + const libp2p::peer::PeerId &peer_id, + const network::vstaging::BackedCandidateAcknowledgement + &acknowledgement) { + REINVOKE(*statements_distribution_thread_handler, + handle_incoming_acknowledgement, + peer_id, + acknowledgement); + + SL_TRACE(logger, + "`BackedCandidateAcknowledgement`. (candidate_hash={})", + acknowledgement.candidate_hash); + const auto &candidate_hash = acknowledgement.candidate_hash; + SL_TRACE(logger, + "Received incoming acknowledgement. (peer={}, candidate hash={})", + peer_id, + candidate_hash); + + TRY_GET_OR_RET(c, candidates.get_confirmed(candidate_hash)); + const RelayHash &relay_parent = c->get().relay_parent(); + const Hash &parent_head_data_hash = c->get().parent_head_data_hash(); + GroupIndex group_index = c->get().group_index(); + ParachainId para_id = c->get().para_id(); + + TRY_GET_OR_RET(opt_parachain_state, tryGetStateByRelayParent(relay_parent)); + auto &relay_parent_state = opt_parachain_state->get(); + + SL_TRACE(logger, + "Handling incoming acknowledgement. (relay_parent={})", + relay_parent); + ManifestImportSuccessOpt x = handle_incoming_manifest_common( + peer_id, + candidate_hash, + relay_parent, + ManifestSummary{ + .claimed_parent_hash = parent_head_data_hash, + .claimed_group_index = group_index, + .statement_knowledge = acknowledgement.statement_knowledge, + }, + para_id, + grid::ManifestKind::Acknowledgement); + CHECK_OR_RET(x); + + SL_TRACE( + logger, "Check local validator. (relay_parent = {})", relay_parent); + CHECK_OR_RET(relay_parent_state.local_validator); + + const auto sender_index = x->sender_index; + auto &local_validator = *relay_parent_state.local_validator; + + SL_TRACE(logger, "Post ack. (relay_parent = {})", relay_parent); + auto messages = post_acknowledgement_statement_messages( + sender_index, + relay_parent, + local_validator.grid_tracker, + relay_parent_state.statement_store, + relay_parent_state.per_session_state->value().groups, + group_index, + candidate_hash, + peer_id, + network::CollationVersion::VStaging); + + SL_TRACE(logger, "Sending messages. (relay_parent = {})", relay_parent); + for (auto &msg : messages) { + if (auto m = if_type(msg)) { + auto message = std::make_shared< + network::WireMessage>( + std::move(m->get())); + network_bridge->send_to_peer( + peer_id, router->getValidationProtocolVStaging(), message); + } else { + assert(false); + } + } + } + + std::deque, + network::VersionedValidatorProtocolMessage>> + StatementDistribution::acknowledgement_and_statement_messages( + const libp2p::peer::PeerId &peer, + network::CollationVersion version, + ValidatorIndex validator_index, + const Groups &groups, + PerRelayParentState &relay_parent_state, + const RelayHash &relay_parent, + GroupIndex group_index, + const CandidateHash &candidate_hash, + const network::vstaging::StatementFilter &local_knowledge) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + if (!relay_parent_state.local_validator) { + return {}; + } + + auto &local_validator = *relay_parent_state.local_validator; + std::deque, + network::VersionedValidatorProtocolMessage>> + messages; + + switch (version) { + case network::CollationVersion::VStaging: { + messages.emplace_back( + std::vector{peer}, + network::VersionedValidatorProtocolMessage{ + network::vstaging::ValidatorProtocolMessage{ + network::vstaging::StatementDistributionMessage{ + network::vstaging::BackedCandidateAcknowledgement{ + .candidate_hash = candidate_hash, + .statement_knowledge = local_knowledge, + }}}}); + } break; + default: { + SL_ERROR(logger, + "Bug ValidationVersion::V1 should not be used in " + "statement-distribution v2, legacy should have handled this"); + return {}; + } break; + }; + + local_validator.grid_tracker.manifest_sent_to( + groups, validator_index, candidate_hash, local_knowledge); + + auto statement_messages = post_acknowledgement_statement_messages( + validator_index, + relay_parent, + local_validator.grid_tracker, + relay_parent_state.statement_store, + groups, + group_index, + candidate_hash, + peer, + version); + + for (auto &&m : statement_messages) { + messages.emplace_back(std::vector{peer}, + std::move(m)); + } + return messages; + } + + void StatementDistribution::handle_incoming_manifest( + const libp2p::peer::PeerId &peer_id, + const network::vstaging::BackedCandidateManifest &manifest) { + REINVOKE(*statements_distribution_thread_handler, + handle_incoming_manifest, + peer_id, + manifest); + + SL_TRACE(logger, + "`BackedCandidateManifest`. (relay_parent={}, " + "candidate_hash={}, para_id={}, parent_head_data_hash={})", + manifest.relay_parent, + manifest.candidate_hash, + manifest.para_id, + manifest.parent_head_data_hash); + + TRY_GET_OR_RET(relay_parent_state, + tryGetStateByRelayParent(manifest.relay_parent)); + + SL_TRACE(logger, + "Handling incoming manifest common. (relay_parent={}, " + "candidate_hash={})", + manifest.relay_parent, + manifest.candidate_hash); + ManifestImportSuccessOpt x = handle_incoming_manifest_common( + peer_id, + manifest.candidate_hash, + manifest.relay_parent, + ManifestSummary{ + .claimed_parent_hash = manifest.parent_head_data_hash, + .claimed_group_index = manifest.group_index, + .statement_knowledge = manifest.statement_knowledge, + }, + manifest.para_id, + grid::ManifestKind::Full); + CHECK_OR_RET(x); + + const auto sender_index = x->sender_index; + if (x->acknowledge) { + SL_TRACE(logger, + "Known candidate - acknowledging manifest. (candidate hash={})", + manifest.candidate_hash); + + SL_TRACE(logger, + "Get groups. (relay_parent={}, candidate_hash={})", + manifest.relay_parent, + manifest.candidate_hash); + auto group = + relay_parent_state->get().per_session_state->value().groups.get( + manifest.group_index); + if (!group) { + return; + } + + network::vstaging::StatementFilter local_knowledge = + local_knowledge_filter(group->size(), + manifest.group_index, + manifest.candidate_hash, + relay_parent_state->get().statement_store); + SL_TRACE(logger, + "Get ack and statement messages. (relay_parent={}, " + "candidate_hash={})", + manifest.relay_parent, + manifest.candidate_hash); + auto messages = acknowledgement_and_statement_messages( + peer_id, + network::CollationVersion::VStaging, + sender_index, + relay_parent_state->get().per_session_state->value().groups, + relay_parent_state->get(), + manifest.relay_parent, + manifest.group_index, + manifest.candidate_hash, + local_knowledge); + + SL_TRACE(logger, + "Send messages. (relay_parent={}, candidate_hash={})", + manifest.relay_parent, + manifest.candidate_hash); + for (auto &[peers, msg] : messages) { + if (auto m = + if_type(msg)) { + auto message = std::make_shared>( + std::move(m->get())); + network_bridge->send_to_peers( + peers, router->getValidationProtocolVStaging(), message); + } else { + assert(false); + } + } + } else if (!candidates.is_confirmed(manifest.candidate_hash)) { + SL_TRACE( + logger, + "Request attested candidate. (relay_parent={}, candidate_hash={})", + manifest.relay_parent, + manifest.candidate_hash); + request_attested_candidate(peer_id, + relay_parent_state->get(), + manifest.relay_parent, + manifest.candidate_hash, + manifest.group_index); + } + } + + std::deque + StatementDistribution::post_acknowledgement_statement_messages( + ValidatorIndex recipient, + const RelayHash &relay_parent, + grid::GridTracker &grid_tracker, + const StatementStore &statement_store, + const Groups &groups, + GroupIndex group_index, + const CandidateHash &candidate_hash, + const libp2p::peer::PeerId &peer, + network::CollationVersion version) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + auto sending_filter = + grid_tracker.pending_statements_for(recipient, candidate_hash); + if (!sending_filter) { + return {}; + } + + std::deque messages; + auto group = groups.get(group_index); + if (!group) { + return messages; + } + + statement_store.groupStatements( + *group, + candidate_hash, + *sending_filter, + [&](const IndexedAndSigned + &statement) { + grid_tracker.sent_or_received_direct_statement(groups, + statement.payload.ix, + recipient, + getPayload(statement), + false); + + switch (version) { + case network::CollationVersion::VStaging: { + messages.emplace_back(network::vstaging::ValidatorProtocolMessage{ + network::vstaging::StatementDistributionMessage{ + network::vstaging::StatementDistributionMessageStatement{ + .relay_parent = relay_parent, + .compact = statement, + }}}); + } break; + default: { + SL_ERROR( + logger, + "Bug ValidationVersion::V1 should not be used in " + "statement-distribution v2, legacy should have handled this"); + } break; + } + }); + return messages; + } + + StatementDistribution::ManifestImportSuccessOpt + StatementDistribution::handle_incoming_manifest_common( + const libp2p::peer::PeerId &peer_id, + const CandidateHash &candidate_hash, + const RelayHash &relay_parent, + ManifestSummary manifest_summary, + ParachainId para_id, + grid::ManifestKind manifest_kind) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + auto peer_state = utils::get(peers, peer_id); + if (!peer_state) { + SL_WARN(logger, "No peer state. (peer_id={})", peer_id); + return {}; + } + + auto relay_parent_state = tryGetStateByRelayParent(relay_parent); + if (!relay_parent_state) { + return {}; + } + + if (!relay_parent_state->get().local_validator) { + return {}; + } + + auto expected_groups = + utils::get(relay_parent_state->get().groups_per_para, para_id); + if (!expected_groups) { + return {}; + } + + if (std::ranges::find(expected_groups->get(), + manifest_summary.claimed_group_index) + == expected_groups->get().end()) { + return {}; + } + + if (!relay_parent_state->get().per_session_state->value().grid_view) { + return {}; + } + + const auto &grid_topology = + *relay_parent_state->get().per_session_state->value().grid_view; + if (manifest_summary.claimed_group_index >= grid_topology.size()) { + return {}; + } + + auto &local_validator = *relay_parent_state->get().local_validator; + auto sender_index = [&]() -> std::optional { + const auto &sub = grid_topology[manifest_summary.claimed_group_index]; + const auto &iter = (manifest_kind == grid::ManifestKind::Full) + ? sub.receiving + : sub.sending; + + if (!iter.empty()) { + return *iter.begin(); + } + return {}; + }(); + + if (!sender_index) { + SL_TRACE(logger, + "Sender not found. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + return {}; + } + + auto group_index = manifest_summary.claimed_group_index; + auto claimed_parent_hash = manifest_summary.claimed_parent_hash; + + auto group = [&]() -> std::span { + if (auto g = + relay_parent_state->get().per_session_state->value().groups.get( + group_index)) { + return *g; + } + return {}; + }(); + + auto disabled_mask = relay_parent_state->get().disabled_bitmask(group); + manifest_summary.statement_knowledge.mask_seconded(disabled_mask); + manifest_summary.statement_knowledge.mask_valid(disabled_mask); + + const auto seconding_limit = relay_parent_state->get().seconding_limit; + + SL_TRACE( + logger, + "Import manifest. (peer_id={}, relay_parent={}, candidate_hash={})", + peer_id, + relay_parent, + candidate_hash); + auto acknowledge_res = local_validator.grid_tracker.import_manifest( + grid_topology, + relay_parent_state->get().per_session_state->value().groups, + candidate_hash, + seconding_limit, + manifest_summary, + manifest_kind, + *sender_index); + + if (acknowledge_res.has_error()) { + SL_WARN(logger, + "Import manifest failed. (peer_id={}, relay_parent={}, " + "candidate_hash={}, error={})", + peer_id, + relay_parent, + candidate_hash, + acknowledge_res.error()); + return {}; + } + + const auto acknowledge = acknowledge_res.value(); + if (!candidates.insert_unconfirmed(peer_id, + candidate_hash, + relay_parent, + group_index, + {{claimed_parent_hash, para_id}})) { + SL_TRACE(logger, + "Insert unconfirmed candidate failed. (candidate hash={}, relay " + "parent={}, para id={}, claimed parent={})", + candidate_hash, + relay_parent, + para_id, + manifest_summary.claimed_parent_hash); + return {}; + } + + if (acknowledge) { + SL_TRACE(logger, + "immediate ack, known candidate. (candidate hash={}, from={}, " + "local_validator={})", + candidate_hash, + *sender_index, + *relay_parent_state->get() + .per_session_state->value() + .local_validator); + } + + return ManifestImportSuccess{ + .acknowledge = acknowledge, + .sender_index = *sender_index, + }; + } + + void StatementDistribution::new_confirmed_candidate_fragment_chain_updates( + const HypotheticalCandidate &candidate) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + fragment_chain_update_inner(std::nullopt, std::nullopt, {candidate}); + } + + void StatementDistribution::new_leaf_fragment_chain_updates( + const Hash &leaf_hash) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + fragment_chain_update_inner({leaf_hash}, std::nullopt, std::nullopt); + } + + void + StatementDistribution::prospective_backed_notification_fragment_chain_updates( + ParachainId para_id, const Hash ¶_head) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + std::pair, ParachainId> p{{para_head}, + para_id}; + fragment_chain_update_inner(std::nullopt, p, std::nullopt); + } + + void StatementDistribution::process_frontier( + std::vector> frontier) { + REINVOKE(*statements_distribution_thread_handler, + process_frontier, + std::move(frontier)); + for (const auto &[hypo, membership] : frontier) { + if (membership.empty()) { + continue; + } + + for (const auto &leaf_hash : membership) { + candidates.note_importable_under(hypo, leaf_hash); + } + + if (auto c = if_type(hypo)) { + auto confirmed_candidate = + candidates.get_confirmed(c->get().candidate_hash); + auto prs = + tryGetStateByRelayParent(c->get().receipt.descriptor.relay_parent); + + if (prs && confirmed_candidate) { + const auto group_index = confirmed_candidate->get().group_index(); + // if (group_index >= session_info.validator_groups.size()) { + // return; + // } + + // Sanity check if group_index is valid for this para at relay parent. + + auto expected_groups = utils::get( + prs->get().groups_per_para, c->get().receipt.descriptor.para_id); + if (!expected_groups) { + continue; + } + + if (std::ranges::find(expected_groups->get(), group_index) + == expected_groups->get().end()) { + continue; + } + + const auto &session_info = + prs->get().per_session_state->value().session_info; + if (group_index >= session_info.validator_groups.size()) { + continue; + } + + const auto &group = session_info.validator_groups[group_index]; + send_backing_fresh_statements( + *confirmed_candidate, + c->get().receipt.descriptor.relay_parent, + prs->get(), + group, + c->get().candidate_hash); + } + } + } + } + + void StatementDistribution::request_hypotetical_membership( + std::vector hypotheticals, + std::optional active_leaf) { + REINVOKE(*main_pool_handler, + request_hypotetical_membership, + std::move(hypotheticals), + active_leaf); + + auto active_leaf_hash = utils::map( + active_leaf, [](const auto &hash) { return std::cref(hash); }); + auto frontier = + prospective_parachains->answer_hypothetical_membership_request( + hypotheticals, active_leaf_hash); + process_frontier(frontier); + } + + void StatementDistribution::fragment_chain_update_inner( + std::optional> active_leaf_hash, + std::optional, ParachainId>> + required_parent_info, + std::optional> + known_hypotheticals) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + std::vector hypotheticals; + if (!known_hypotheticals) { + hypotheticals = candidates.frontier_hypotheticals(required_parent_info); + } else { + hypotheticals.emplace_back(known_hypotheticals->get()); + } + + auto active_leaf = + utils::map(active_leaf_hash, + [](const auto &ref_hash) -> Hash { return ref_hash.get(); }); + request_hypotetical_membership(std::move(hypotheticals), active_leaf); + } + + void StatementDistribution::provide_candidate_to_grid( + const CandidateHash &candidate_hash, + PerRelayParentState &relay_parent_state, + const ConfirmedCandidate &confirmed_candidate, + const runtime::SessionInfo &session_info) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + CHECK_OR_RET(relay_parent_state.local_validator); + auto &local_validator = *relay_parent_state.local_validator; + + const auto relay_parent = confirmed_candidate.relay_parent(); + const auto group_index = confirmed_candidate.group_index(); + + if (!relay_parent_state.per_session_state->value().grid_view) { + SL_TRACE(logger, + "Cannot handle backable candidate due to lack of topology. " + "(candidate={}, relay_parent={})", + candidate_hash, + relay_parent); + return; + } + + const auto &grid_view = + *relay_parent_state.per_session_state->value().grid_view; + const auto group = + relay_parent_state.per_session_state->value().groups.get(group_index); + if (!group) { + SL_TRACE(logger, + "Handled backed candidate with unknown group? (candidate={}, " + "relay_parent={}, group_index={})", + candidate_hash, + relay_parent, + group_index); + return; + } + + const auto group_size = group->size(); + auto filter = local_knowledge_filter(group_size, + group_index, + candidate_hash, + relay_parent_state.statement_store); + + auto actions = local_validator.grid_tracker.add_backed_candidate( + grid_view, candidate_hash, group_index, filter); + + network::vstaging::BackedCandidateManifest manifest{ + .relay_parent = relay_parent, + .candidate_hash = candidate_hash, + .group_index = group_index, + .para_id = confirmed_candidate.para_id(), + .parent_head_data_hash = confirmed_candidate.parent_head_data_hash(), + .statement_knowledge = filter}; + + network::vstaging::BackedCandidateAcknowledgement acknowledgement{ + .candidate_hash = candidate_hash, .statement_knowledge = filter}; + + std::vector manifest_peers; + std::vector ack_peers; + std::deque, + network::VersionedValidatorProtocolMessage>> + post_statements; + + for (const auto &[v, action] : actions) { + auto peer_opt = query_audi->get(session_info.discovery_keys[v]); + if (!peer_opt) { + SL_TRACE(logger, + "No peer info. (relay_parent={}, validator_index={}, " + "candidate_hash={})", + relay_parent, + v, + candidate_hash); + continue; + } + + auto peer_state = utils::get(peers, peer_opt->id); + if (!peer_state) { + SL_TRACE(logger, + "No peer state. (relay_parent={}, peer={}, candidate_hash={})", + relay_parent, + peer_opt->id, + candidate_hash); + continue; + } + + if (!peer_state->get().knows_relay_parent(relay_parent)) { + SL_TRACE(logger, + "Peer doesn't know relay parent. (relay_parent={}, peer={}, " + "candidate_hash={})", + relay_parent, + peer_opt->id, + candidate_hash); + continue; + } + + switch (action) { + case grid::ManifestKind::Full: { + SL_TRACE(logger, "Full manifest -> {}", v); + manifest_peers.emplace_back(peer_opt->id); + } break; + case grid::ManifestKind::Acknowledgement: { + SL_TRACE(logger, "Ack manifest -> {}", v); + ack_peers.emplace_back(peer_opt->id); + } break; + } + + SL_TRACE( + logger, + "Mark message sent to for RP. (relay_parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + local_validator.grid_tracker.manifest_sent_to( + relay_parent_state.per_session_state->value().groups, + v, + candidate_hash, + filter); + + auto msgs = post_acknowledgement_statement_messages( + v, + relay_parent, + local_validator.grid_tracker, + relay_parent_state.statement_store, + relay_parent_state.per_session_state->value().groups, + group_index, + candidate_hash, + peer_opt->id, + network::CollationVersion::VStaging); + + for (auto &msg : msgs) { + post_statements.emplace_back( + std::vector{peer_opt->id}, std::move(msg)); + } + } + + if (!manifest_peers.empty()) { + SL_TRACE(logger, + "Sending manifest to v2 peers. (candidate_hash={}, n_peers={})", + candidate_hash, + manifest_peers.size()); + auto message = std::make_shared< + network::WireMessage>( + kagome::network::vstaging::ValidatorProtocolMessage{ + kagome::network::vstaging::StatementDistributionMessage{ + manifest}}); + + network_bridge->send_to_peers( + manifest_peers, router->getValidationProtocolVStaging(), message); + } + + if (!ack_peers.empty()) { + SL_TRACE(logger, + "Sending acknowledgement to v2 peers. (candidate_hash={}, " + "n_peers={})", + candidate_hash, + ack_peers.size()); + auto message = std::make_shared< + network::WireMessage>( + kagome::network::vstaging::ValidatorProtocolMessage{ + kagome::network::vstaging::StatementDistributionMessage{ + acknowledgement}}); + network_bridge->send_to_peers( + ack_peers, router->getValidationProtocolVStaging(), message); + } + + if (!post_statements.empty()) { + SL_TRACE( + logger, + "Sending statements to v2 peers. (candidate_hash={}, n_peers={})", + candidate_hash, + post_statements.size()); + + for (auto &[peers, msg] : post_statements) { + if (auto m = + if_type(msg)) { + auto message = std::make_shared>( + std::move(m->get())); + network_bridge->send_to_peers( + peers, router->getValidationProtocolVStaging(), message); + } else { + assert(false); + } + } + } + } + + void StatementDistribution::send_backing_fresh_statements( + const ConfirmedCandidate &confirmed, + const RelayHash &relay_parent, + PerRelayParentState &per_relay_parent, + const std::vector &group, + const CandidateHash &candidate_hash) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + std::vector> + imported; + per_relay_parent.statement_store.fresh_statements_for_backing( + group, + candidate_hash, + [&](const IndexedAndSigned + &statement) { + const auto &v = statement.payload.ix; + const auto &compact = getPayload(statement); + imported.emplace_back(v, compact); + + SignedFullStatementWithPVD carrying_pvd{ + .payload = + { + .payload = visit_in_place( + compact.inner_value, + [&](const network::vstaging::SecondedCandidateHash &) + -> StatementWithPVD { + return StatementWithPVDSeconded{ + .committed_receipt = confirmed.receipt, + .pvd = confirmed.persisted_validation_data, + }; + }, + [](const network::vstaging::ValidCandidateHash &val) + -> StatementWithPVD { + return StatementWithPVDValid{ + .candidate_hash = val.hash, + }; + }, + [](const auto &) -> StatementWithPVD { + UNREACHABLE; + }), + .ix = statement.payload.ix, + }, + .signature = statement.signature, + }; + + if (auto parachain_proc = parachain_processor.lock()) { + SL_TRACE(logger, "Handle statement {}", relay_parent); + parachain_proc->handleStatement(relay_parent, carrying_pvd); + } + }); + + for (const auto &[v, s] : imported) { + per_relay_parent.statement_store.note_known_by_backing(v, s); + } + } + + outcome::result> + StatementDistribution::handle_cluster_statement( + const RelayHash &relay_parent, + ClusterTracker &cluster_tracker, + SessionIndex session, + const runtime::SessionInfo &session_info, + const network::vstaging::SignedCompactStatement &statement, + ValidatorIndex cluster_sender_index) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + const auto accept = cluster_tracker.can_receive( + cluster_sender_index, + statement.payload.ix, + network::vstaging::from(getPayload(statement))); + if (accept != outcome::success(Accept::Ok) + && accept != outcome::success(Accept::WithPrejudice)) { + SL_ERROR(logger, "Reject outgoing error."); + return Error::CLUSTER_TRACKER_ERROR; + } + OUTCOME_TRY(check_statement_signature( + session, session_info.validators, relay_parent, statement)); + + cluster_tracker.note_received( + cluster_sender_index, + statement.payload.ix, + network::vstaging::from(getPayload(statement))); + + const auto should_import = (outcome::success(Accept::Ok) == accept); + if (should_import) { + return statement; + } + return std::nullopt; + } + + void StatementDistribution::handle_incoming_statement( + const libp2p::peer::PeerId &peer_id, + const network::vstaging::StatementDistributionMessageStatement &stm) { + REINVOKE(*statements_distribution_thread_handler, + handle_incoming_statement, + peer_id, + stm); + SL_TRACE(logger, + "`StatementDistributionMessageStatement`. (relay_parent={}, " + "candidate_hash={})", + stm.relay_parent, + candidateHash(getPayload(stm.compact))); + auto parachain_state = tryGetStateByRelayParent(stm.relay_parent); + if (!parachain_state) { + SL_TRACE(logger, + "After request pov no parachain state on relay_parent. (relay " + "parent={})", + stm.relay_parent); + return; + } + + const auto &session_info = + parachain_state->get().per_session_state->value().session_info; + if (parachain_state->get().is_disabled(stm.compact.payload.ix)) { + SL_TRACE( + logger, + "Ignoring a statement from disabled validator. (relay parent={}, " + "validator={})", + stm.relay_parent, + stm.compact.payload.ix); + return; + } + + CHECK_OR_RET(parachain_state->get().local_validator); + auto &local_validator = *parachain_state->get().local_validator; + auto originator_group = + parachain_state->get() + .per_session_state->value() + .groups.byValidatorIndex(stm.compact.payload.ix); + if (!originator_group) { + SL_TRACE(logger, + "No correct validator index in statement. (relay parent={}, " + "validator={})", + stm.relay_parent, + stm.compact.payload.ix); + return; + } + + auto &active = local_validator.active; + auto cluster_sender_index = [&]() -> std::optional { + std::span allowed_senders; + if (active) { + allowed_senders = active->cluster_tracker.senders_for_originator( + stm.compact.payload.ix); + } + + if (auto peer = query_audi->get(peer_id)) { + for (const auto i : allowed_senders) { + if (i < session_info.discovery_keys.size() + && *peer == session_info.discovery_keys[i]) { + return i; + } + } + } + return std::nullopt; + }(); + + if (active && cluster_sender_index) { + if (handle_cluster_statement( + stm.relay_parent, + active->cluster_tracker, + parachain_state->get().per_session_state->value().session, + parachain_state->get().per_session_state->value().session_info, + stm.compact, + *cluster_sender_index) + .has_error()) { + return; + } + } else { + std::optional> grid_sender_index; + for (const auto &[i, validator_knows_statement] : + local_validator.grid_tracker.direct_statement_providers( + parachain_state->get().per_session_state->value().groups, + stm.compact.payload.ix, + getPayload(stm.compact))) { + if (i >= session_info.discovery_keys.size()) { + continue; + } + + /// TODO(iceseer): do check is authority + /// const auto &ad = opt_session_info->discovery_keys[i]; + grid_sender_index.emplace(i, validator_knows_statement); + break; + } + + CHECK_OR_RET(grid_sender_index); + const auto &[gsi, validator_knows_statement] = *grid_sender_index; + + CHECK_OR_RET(!validator_knows_statement); + if (handle_grid_statement(stm.relay_parent, + parachain_state->get(), + local_validator.grid_tracker, + stm.compact, + gsi) + .has_error()) { + return; + } + } + + const auto &statement = getPayload(stm.compact); + const auto originator_index = stm.compact.payload.ix; + const auto &candidate_hash = candidateHash(getPayload(stm.compact)); + const bool res = candidates.insert_unconfirmed(peer_id, + candidate_hash, + stm.relay_parent, + *originator_group, + std::nullopt); + CHECK_OR_RET(res); + const auto confirmed = candidates.get_confirmed(candidate_hash); + const auto is_confirmed = candidates.is_confirmed(candidate_hash); + const auto &group = session_info.validator_groups[*originator_group]; + + if (!is_confirmed) { + request_attested_candidate(peer_id, + parachain_state->get(), + stm.relay_parent, + candidate_hash, + *originator_group); + } + + /// TODO(iceseer): do https://github.com/qdrvm/kagome/issues/1888 + /// check statement signature + + const auto was_fresh_opt = parachain_state->get().statement_store.insert( + parachain_state->get().per_session_state->value().groups, + stm.compact, + StatementOrigin::Remote); + if (!was_fresh_opt) { + SL_WARN(logger, + "Accepted message from unknown validator. (relay parent={}, " + "validator={})", + stm.relay_parent, + stm.compact.payload.ix); + return; + } + + if (!*was_fresh_opt) { + SL_TRACE(logger, + "Statement was not fresh. (relay parent={}, validator={})", + stm.relay_parent, + stm.compact.payload.ix); + return; + } + + const auto is_importable = candidates.is_importable(candidate_hash); + if (parachain_state->get().per_session_state->value().grid_view) { + local_validator.grid_tracker.learned_fresh_statement( + parachain_state->get().per_session_state->value().groups, + *parachain_state->get().per_session_state->value().grid_view, + originator_index, + statement); + } + + if (is_importable && confirmed) { + send_backing_fresh_statements(confirmed->get(), + stm.relay_parent, + parachain_state->get(), + group, + candidate_hash); + } + + circulate_statement(stm.relay_parent, parachain_state->get(), stm.compact); + } + + outcome::result< + std::reference_wrapper> + StatementDistribution::check_statement_signature( + SessionIndex session_index, + const std::vector &validators, + const RelayHash &relay_parent, + const network::vstaging::SignedCompactStatement &statement) { + OUTCOME_TRY(signing_context, + SigningContext::make(parachain_host, relay_parent)); + OUTCOME_TRY(verified, + crypto_provider->verify( + statement.signature, + signing_context.signable(*hasher, getPayload(statement)), + validators[statement.payload.ix])); + + if (!verified) { + return Error::INCORRECT_SIGNATURE; + } + return std::cref(statement); + } + + void StatementDistribution::circulate_statement( + const RelayHash &relay_parent, + PerRelayParentState &relay_parent_state, + const IndexedAndSigned &statement) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + const auto &session_info = + relay_parent_state.per_session_state->value().session_info; + const auto &compact_statement = getPayload(statement); + const auto &candidate_hash = candidateHash(compact_statement); + const auto originator = statement.payload.ix; + const auto is_confirmed = candidates.is_confirmed(candidate_hash); + + CHECK_OR_RET(relay_parent_state.local_validator); + enum DirectTargetKind : uint8_t { + Cluster, + Grid, + }; + + auto &local_validator = *relay_parent_state.local_validator; + auto targets = + [&]() -> std::vector> { + auto statement_group = + relay_parent_state.per_session_state->value().groups.byValidatorIndex( + originator); + + bool cluster_relevant = false; + std::vector> targets; + std::span all_cluster_targets; + + if (local_validator.active) { + auto &active = *local_validator.active; + cluster_relevant = + (statement_group && *statement_group == active.group); + if (is_confirmed && cluster_relevant) { + for (const auto v : active.cluster_tracker.targets()) { + if (auto res = active.cluster_tracker.can_send( + v, originator, network::vstaging::from(compact_statement)); + res.has_error()) { + SL_TRACE(logger, + "Skip cluster target because of `can_send`. " + "(relay_parent={}, error={})", + relay_parent, + (uint32_t)res.error()); + continue; + } + if (v == active.index) { + SL_TRACE(logger, + "Skip cluster target because of index eq. " + "(relay_parent={}, v={})", + relay_parent, + v); + continue; + } + if (v >= session_info.discovery_keys.size()) { + SL_TRACE(logger, + "Skip cluster target because index is out of bound. " + "(relay_parent={}, v={}, discovery_keys={})", + relay_parent, + v, + session_info.discovery_keys.size()); + continue; + } + targets.emplace_back(v, DirectTargetKind::Cluster); + } + } else { + SL_TRACE(logger, + "Skip target because of either. (relay_parent={}, " + "is_confirmed={}, cluster_relevant={})", + relay_parent, + is_confirmed ? "[yes]" : "[no]", + cluster_relevant ? "[yes]" : "[no]"); + } + all_cluster_targets = active.cluster_tracker.targets(); + } + + for (const auto v : local_validator.grid_tracker.direct_statement_targets( + relay_parent_state.per_session_state->value().groups, + originator, + compact_statement)) { + const auto can_use_grid = !cluster_relevant + || std::ranges::find(all_cluster_targets, v) + == all_cluster_targets.end(); + if (!can_use_grid) { + SL_TRACE( + logger, + "Skip grid target because can not use grid. (relay_parent={})", + relay_parent); + continue; + } + if (v >= session_info.discovery_keys.size()) { + SL_TRACE(logger, + "Skip grid target because index is out of bound. " + "(relay_parent={}, v={}, discovery_keys={})", + relay_parent, + v, + session_info.discovery_keys.size()); + continue; + } + targets.emplace_back(v, DirectTargetKind::Grid); + } + + return targets; + }(); + SL_TRACE(logger, + "Formed resulted targets list. (relay_parent={}, size={})", + relay_parent, + targets.size()); + + std::vector statement_to_peers; + for (const auto &[target, kind] : targets) { + auto peer = query_audi->get(session_info.discovery_keys[target]); + if (!peer) { + SL_TRACE(logger, + "Skip send statement to validator. No audi. (relay_parent={}, " + "discovery_key={})", + relay_parent, + session_info.discovery_keys[target]); + continue; + } + + auto peer_state = utils::get(peers, peer->id); + if (!peer_state) { + SL_TRACE(logger, + "Skip send statement to validator. No peer. (relay_parent={}, " + "peer={})", + relay_parent, + peer->id); + continue; + } + + if (!peer_state->get().knows_relay_parent(relay_parent)) { + SL_TRACE(logger, + "Skip send statement to validator. Doesn't know relay_parent. " + "(relay_parent={}, peer={})", + relay_parent, + peer->id); + continue; + } + + switch (kind) { + case Cluster: { + auto &active = *local_validator.active; + if (auto res = active.cluster_tracker.can_send( + target, + originator, + network::vstaging::from(compact_statement)); + res.has_value()) { + active.cluster_tracker.note_sent( + target, originator, network::vstaging::from(compact_statement)); + statement_to_peers.emplace_back(peer->id); + } else { + SL_TRACE(logger, + "Skip cluster peer send because of `can_send`. " + "(relay_parent={}, error={})", + relay_parent, + (uint32_t)res.error()); + } + } break; + case Grid: { + statement_to_peers.emplace_back(peer->id); + local_validator.grid_tracker.sent_or_received_direct_statement( + relay_parent_state.per_session_state->value().groups, + originator, + target, + compact_statement, + false); + } break; + } + } + + auto message_v2 = std::make_shared< + network::WireMessage>( + kagome::network::vstaging::ValidatorProtocolMessage{ + kagome::network::vstaging::StatementDistributionMessage{ + kagome::network::vstaging:: + StatementDistributionMessageStatement{ + .relay_parent = relay_parent, + .compact = statement, + }}}); + SL_TRACE( + logger, + "Send statements to validators. (relay_parent={}, validators_count={})", + relay_parent, + statement_to_peers.size()); + network_bridge->send_to_peers(statement_to_peers, + router->getValidationProtocolVStaging(), + message_v2); + } + + void StatementDistribution::share_local_statement( + const primitives::BlockHash &relay_parent, + const SignedFullStatementWithPVD &statement) { + REINVOKE(*statements_distribution_thread_handler, + share_local_statement, + relay_parent, + statement); + + auto per_relay_parent = tryGetStateByRelayParent(relay_parent); + const CandidateHash candidate_hash = + candidateHashFrom(getPayload(statement), hasher); + + SL_TRACE(logger, + "Sharing statement. (relay parent={}, candidate hash={}, " + "statement_ix={})", + relay_parent, + candidate_hash, + statement.payload.ix); + + auto validator_state = per_relay_parent->get().active_validator_state(); + if (!validator_state) { + SL_WARN(logger, + "Invalid share statement. (relay parent={}, candidate_hash={})", + relay_parent, + candidate_hash); + return; + } + + const network::ValidatorIndex local_index = validator_state->get().index; + const GroupIndex local_group = validator_state->get().group; + const auto local_assignment = validator_state->get().assignment; + + const Groups &groups = + per_relay_parent->get().per_session_state->value().groups; + // const std::optional &local_assignment = + // per_relay_parent.assigned_para; + // const network::ValidatorIndex local_index = *per_relay_parent.our_index; + // const auto local_group_opt = groups.byValidatorIndex(local_index); + // const GroupIndex local_group = *local_group_opt; + + std::optional> expected = visit_in_place( + getPayload(statement), + [&](const StatementWithPVDSeconded &v) + -> std::optional> { + return std::make_pair(v.committed_receipt.descriptor.para_id, + v.committed_receipt.descriptor.relay_parent); + }, + [&](const StatementWithPVDValid &v) + -> std::optional> { + if (auto p = candidates.get_confirmed(v.candidate_hash)) { + return std::make_pair(p->get().para_id(), p->get().relay_parent()); + } + return std::nullopt; + }); + const bool is_seconded = + is_type(getPayload(statement)); + + if (!expected) { + SL_ERROR( + logger, "Invalid share statement. (relay parent={})", relay_parent); + return; + } + const auto &[expected_para, expected_relay_parent] = *expected; + + if (local_index != statement.payload.ix) { + SL_ERROR(logger, + "Invalid share statement because of validator index. (relay " + "parent={})", + relay_parent); + return; + } + + const auto seconding_limit = per_relay_parent->get().seconding_limit; + if (is_seconded + && per_relay_parent->get().statement_store.seconded_count(local_index) + == seconding_limit) { + SL_WARN( + logger, + "Local node has issued too many `Seconded` statements. (limit={})", + seconding_limit); + return; + } + + if (!local_assignment || *local_assignment != expected_para + || relay_parent != expected_relay_parent) { + SL_ERROR( + logger, + "Invalid share statement because local assignment. (relay parent={})", + relay_parent); + return; + } + + IndexedAndSigned compact_statement = + signed_to_compact(statement, hasher); + std::optional post_confirmation; + if (auto s = + if_type(getPayload(statement))) { + post_confirmation = + candidates.confirm_candidate(candidate_hash, + s->get().committed_receipt, + s->get().pvd, + local_group, + hasher); + } + + if (auto r = per_relay_parent->get().statement_store.insert( + groups, compact_statement, StatementOrigin::Local); + !r || !*r) { + SL_ERROR(logger, + "Invalid share statement because statement store insertion " + "failed. (relay parent={})", + relay_parent); + return; + } + + if (per_relay_parent->get().local_validator + && per_relay_parent->get().local_validator->active) { + per_relay_parent->get() + .local_validator->active->cluster_tracker.note_issued( + local_index, + network::vstaging::from(getPayload(compact_statement))); + } + + if (per_relay_parent->get().per_session_state->value().grid_view) { + auto &l = *per_relay_parent->get().local_validator; + l.grid_tracker.learned_fresh_statement( + groups, + *per_relay_parent->get().per_session_state->value().grid_view, + local_index, + getPayload(compact_statement)); + } + + circulate_statement( + relay_parent, per_relay_parent->get(), compact_statement); + if (post_confirmation) { + apply_post_confirmation(*post_confirmation); + } + } + + void StatementDistribution::send_pending_grid_messages( + const RelayHash &relay_parent, + const libp2p::peer::PeerId &peer_id, + network::CollationVersion version, + ValidatorIndex peer_validator_id, + const Groups &groups, + PerRelayParentState &relay_parent_state) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + CHECK_OR_RET(relay_parent_state.local_validator); + + auto pending_manifests = + relay_parent_state.local_validator->grid_tracker.pending_manifests_for( + peer_validator_id); + std::deque, + network::VersionedValidatorProtocolMessage>> + messages; + for (const auto &[candidate_hash, kind] : pending_manifests) { + const auto confirmed_candidate = candidates.get_confirmed(candidate_hash); + if (!confirmed_candidate) { + continue; + } + + const auto group_index = confirmed_candidate->get().group_index(); + TRY_GET_OR_RET(group, groups.get(group_index)); + + const auto group_size = group->size(); + auto local_knowledge = + local_knowledge_filter(group_size, + group_index, + candidate_hash, + relay_parent_state.statement_store); + + switch (kind) { + case grid::ManifestKind::Full: { + const network::vstaging::BackedCandidateManifest manifest{ + .relay_parent = relay_parent, + .candidate_hash = candidate_hash, + .group_index = group_index, + .para_id = confirmed_candidate->get().para_id(), + .parent_head_data_hash = + confirmed_candidate->get().parent_head_data_hash(), + .statement_knowledge = local_knowledge, + }; + + auto &grid = relay_parent_state.local_validator->grid_tracker; + grid.manifest_sent_to( + groups, peer_validator_id, candidate_hash, local_knowledge); + + switch (version) { + case network::CollationVersion::VStaging: { + messages.emplace_back( + std::vector{peer_id}, + network::VersionedValidatorProtocolMessage{ + kagome::network::vstaging::ValidatorProtocolMessage{ + kagome::network::vstaging:: + StatementDistributionMessage{manifest}}}); + } break; + default: { + SL_ERROR(logger, + "Bug ValidationVersion::V1 should not be used in " + "statement-distribution v2, legacy should have handled " + "this."); + } break; + }; + } break; + case grid::ManifestKind::Acknowledgement: { + auto m = acknowledgement_and_statement_messages( + peer_id, + network::CollationVersion::VStaging, + peer_validator_id, + groups, + relay_parent_state, + relay_parent, + group_index, + candidate_hash, + local_knowledge); + messages.insert(messages.end(), + std::move_iterator(m.begin()), + std::move_iterator(m.end())); + + } break; + } + } + + { + auto &grid_tracker = relay_parent_state.local_validator->grid_tracker; + auto pending_statements = + grid_tracker.all_pending_statements_for(peer_validator_id); + + for (const auto &[originator, compact] : pending_statements) { + auto res = pending_statement_network_message( + relay_parent_state.statement_store, + relay_parent, + peer_id, + network::CollationVersion::VStaging, + originator, + compact); + + if (res) { + grid_tracker.sent_or_received_direct_statement( + groups, originator, peer_validator_id, compact, false); + + messages.emplace_back(std::move(*res)); + } + } + } + + for (auto &[peers, msg] : messages) { + if (auto m = if_type(msg)) { + auto message = std::make_shared< + network::WireMessage>( + std::move(m->get())); + network_bridge->send_to_peers( + peers, router->getValidationProtocolVStaging(), message); + } else { + assert(false); + } + } + } + + void StatementDistribution::send_pending_cluster_statements( + const RelayHash &relay_parent, + const libp2p::peer::PeerId &peer_id, + network::CollationVersion version, + ValidatorIndex peer_validator_id, + PerRelayParentState &relay_parent_state) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + CHECK_OR_RET(relay_parent_state.local_validator); + CHECK_OR_RET(relay_parent_state.local_validator->active); + + const auto pending_statements = + relay_parent_state.local_validator->active->cluster_tracker + .pending_statements_for(peer_validator_id); + std::deque, + network::VersionedValidatorProtocolMessage>> + messages; + for (const auto &[originator, compact] : pending_statements) { + if (!candidates.is_confirmed(candidateHash(compact))) { + continue; + } + + auto res = + pending_statement_network_message(relay_parent_state.statement_store, + relay_parent, + peer_id, + version, + originator, + network::vstaging::from(compact)); + + if (res) { + relay_parent_state.local_validator->active->cluster_tracker.note_sent( + peer_validator_id, originator, compact); + messages.emplace_back(*res); + } + } + + for (auto &[peers, msg] : messages) { + if (auto m = if_type(msg)) { + auto message = std::make_shared< + network::WireMessage>( + std::move(m->get())); + network_bridge->send_to_peers( + peers, router->getValidationProtocolVStaging(), message); + } else { + BOOST_ASSERT(false); + } + } + } + + std::optional, + network::VersionedValidatorProtocolMessage>> + StatementDistribution::pending_statement_network_message( + const StatementStore &statement_store, + const RelayHash &relay_parent, + const libp2p::peer::PeerId &peer, + network::CollationVersion version, + ValidatorIndex originator, + const network::vstaging::CompactStatement &compact) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + switch (version) { + case network::CollationVersion::VStaging: { + auto s = statement_store.validator_statement(originator, compact); + if (s) { + return std::make_pair( + std::vector{peer}, + network::VersionedValidatorProtocolMessage{ + network::vstaging::ValidatorProtocolMessage{ + network::vstaging::StatementDistributionMessage{ + network::vstaging:: + StatementDistributionMessageStatement{ + .relay_parent = relay_parent, + .compact = s->get().statement, + }}}}); + } + } break; + default: { + SL_ERROR(logger, + "Bug ValidationVersion::V1 should not be used in " + "statement-distribution v2, legacy should have handled this"); + } break; + } + return {}; + } + + void StatementDistribution::send_peer_messages_for_relay_parent( + const libp2p::peer::PeerId &peer_id, const RelayHash &relay_parent) { + REINVOKE(*statements_distribution_thread_handler, + send_peer_messages_for_relay_parent, + peer_id, + relay_parent); + TRY_GET_OR_RET(parachain_state, tryGetStateByRelayParent(relay_parent)); + + const network::CollationVersion version = + network::CollationVersion::VStaging; + if (auto auth_id = query_audi->get(peer_id)) { + if (auto vi = utils::get(parachain_state->get() + .per_session_state->value() + .authority_lookup, + *auth_id)) { + SL_TRACE(logger, + "Send pending cluster/grid messages. (peer={}. validator " + "index={}, relay_parent={})", + peer_id, + vi->get(), + relay_parent); + send_pending_cluster_statements( + relay_parent, peer_id, version, vi->get(), parachain_state->get()); + + send_pending_grid_messages( + relay_parent, + peer_id, + version, + vi->get(), + parachain_state->get().per_session_state->value().groups, + parachain_state->get()); + } + } + } + + std::optional + StatementDistribution::find_active_validator_state( + ValidatorIndex validator_index, + const Groups &groups, + const std::vector &availability_cores, + const runtime::GroupDescriptor &group_rotation_info, + const std::optional &maybe_claim_queue, + size_t seconding_limit, + size_t max_candidate_depth) { + BOOST_ASSERT(statements_distribution_thread_handler->isInCurrentThread()); + + if (groups.all_empty()) { + return std::nullopt; + } + + const auto our_group = groups.byValidatorIndex(validator_index); + if (!our_group) { + return std::nullopt; + } + + const auto core_index = + group_rotation_info.coreForGroup(*our_group, availability_cores.size()); + std::optional para_assigned_to_core; + if (maybe_claim_queue) { + para_assigned_to_core = maybe_claim_queue->get_claim_for(core_index, 0); + } else { + if (core_index < availability_cores.size()) { + const auto &core_state = availability_cores[core_index]; + visit_in_place( + core_state, + [&](const runtime::ScheduledCore &scheduled) { + para_assigned_to_core = scheduled.para_id; + }, + [&](const runtime::OccupiedCore &occupied) { + if (max_candidate_depth >= 1 && occupied.next_up_on_available) { + para_assigned_to_core = occupied.next_up_on_available->para_id; + } + }, + [](const auto &) {}); + } + } + + const auto group_validators = groups.get(*our_group); + if (!group_validators) { + return std::nullopt; + } + + return LocalValidatorState{ + .grid_tracker = {}, + .active = + ActiveValidatorState{ + .index = validator_index, + .group = *our_group, + .assignment = para_assigned_to_core, + .cluster_tracker = ClusterTracker( + {group_validators->begin(), group_validators->end()}, + seconding_limit), + }, + }; + } + +} // namespace kagome::parachain::statement_distribution diff --git a/core/parachain/validator/statement_distribution/statement_distribution.hpp b/core/parachain/validator/statement_distribution/statement_distribution.hpp new file mode 100644 index 0000000000..6f72effd13 --- /dev/null +++ b/core/parachain/validator/statement_distribution/statement_distribution.hpp @@ -0,0 +1,389 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include "authority_discovery/query/query.hpp" +#include "common/ref_cache.hpp" +#include "consensus/babe/babe_config_repository.hpp" +#include "consensus/babe/impl/babe_digests_util.hpp" +#include "consensus/timeline/slots_util.hpp" +#include "network/can_disconnect.hpp" +#include "network/peer_manager.hpp" +#include "network/peer_view.hpp" +#include "network/router.hpp" +#include "parachain/approval/approval_thread_pool.hpp" +#include "parachain/validator/impl/candidates.hpp" +#include "parachain/validator/network_bridge.hpp" +#include "parachain/validator/signer.hpp" +#include "parachain/validator/statement_distribution/peer_state.hpp" +#include "parachain/validator/statement_distribution/per_session_state.hpp" +#include "parachain/validator/statement_distribution/types.hpp" +#include "utils/pool_handler_ready_make.hpp" + +namespace kagome::parachain { + struct ParachainProcessorImpl; +} + +namespace kagome::parachain::statement_distribution { + + struct StatementDistribution + : std::enable_shared_from_this, + network::CanDisconnect { + enum class Error : uint8_t { + RESPONSE_ALREADY_RECEIVED = 1, + COLLATION_NOT_FOUND, + KEY_NOT_PRESENT, + VALIDATION_FAILED, + VALIDATION_SKIPPED, + OUT_OF_VIEW, + DUPLICATE, + NO_INSTANCE, + NOT_A_VALIDATOR, + NOT_SYNCHRONIZED, + UNDECLARED_COLLATOR, + PEER_LIMIT_REACHED, + PROTOCOL_MISMATCH, + NOT_CONFIRMED, + NO_STATE, + NO_SESSION_INFO, + OUT_OF_BOUND, + REJECTED_BY_PROSPECTIVE_PARACHAINS, + INCORRECT_BITFIELD_SIZE, + CORE_INDEX_UNAVAILABLE, + INCORRECT_SIGNATURE, + CLUSTER_TRACKER_ERROR, + PERSISTED_VALIDATION_DATA_NOT_FOUND, + PERSISTED_VALIDATION_DATA_MISMATCH, + CANDIDATE_HASH_MISMATCH, + PARENT_HEAD_DATA_MISMATCH, + NO_PEER, + ALREADY_REQUESTED, + NOT_ADVERTISED, + WRONG_PARA, + THRESHOLD_LIMIT_REACHED, + }; + + StatementDistribution( + std::shared_ptr sf, + std::shared_ptr app_state_manager, + StatementDistributionThreadPool &statements_distribution_thread_pool, + std::shared_ptr prospective_parachains, + std::shared_ptr parachain_host, + std::shared_ptr block_tree, + std::shared_ptr query_audi, + std::shared_ptr network_bridge, + std::shared_ptr router, + common::MainThreadPool &main_thread_pool, + std::shared_ptr hasher, + std::shared_ptr crypto_provider, + std::shared_ptr peer_view, + LazySPtr slots_util, + std::shared_ptr babe_config_repo, + primitives::events::PeerSubscriptionEnginePtr peer_events_engine); + + void request_attested_candidate(const libp2p::peer::PeerId &peer, + PerRelayParentState &relay_parent_state, + const RelayHash &relay_parent, + const CandidateHash &candidate_hash, + GroupIndex group_index); + + // outcome::result + void OnFetchAttestedCandidateRequest( + const network::vstaging::AttestedCandidateRequest &request, + std::shared_ptr stream); + + // CanDisconnect + bool can_disconnect(const libp2p::PeerId &) const override; + void store_parachain_processor(std::weak_ptr pp) { + BOOST_ASSERT(!pp.expired()); + parachain_processor = std::move(pp); + } + bool tryStart(); + + // Handles BackedCandidateManifest message + // It performs various checks and operations, and if everything is + // successful, it sends acknowledgement and statement messages to the + // validators group or sends a request to fetch the attested candidate. + void handle_incoming_manifest( + const libp2p::peer::PeerId &peer_id, + const network::vstaging::BackedCandidateManifest &msg); + + void handle_incoming_acknowledgement( + const libp2p::peer::PeerId &peer_id, + const network::vstaging::BackedCandidateAcknowledgement + &acknowledgement); + + void handle_incoming_statement( + const libp2p::peer::PeerId &peer_id, + const network::vstaging::StatementDistributionMessageStatement &stm); + + void handle_backed_candidate_message(const CandidateHash &candidate_hash); + + void share_local_statement(const primitives::BlockHash &relay_parent, + const SignedFullStatementWithPVD &statement); + + private: + struct ManifestImportSuccess { + bool acknowledge; + ValidatorIndex sender_index; + }; + using ManifestImportSuccessOpt = std::optional; + using ManifestSummary = parachain::grid::ManifestSummary; + + struct RelayParentContext { + Hash relay_parent; + std::optional validator_index; + std::optional v_index; + }; + + std::optional> + tryGetStateByRelayParent(const primitives::BlockHash &relay_parent); + outcome::result> + getStateByRelayParent(const primitives::BlockHash &relay_parent); + + void handle_response( + outcome::result &&r, + const RelayHash &relay_parent, + const CandidateHash &candidate_hash, + GroupIndex group_index); + + std::optional find_active_validator_state( + ValidatorIndex validator_index, + const Groups &groups, + const std::vector &availability_cores, + const runtime::GroupDescriptor &group_rotation_info, + const std::optional &maybe_claim_queue, + size_t seconding_limit, + size_t max_candidate_depth); + + void handle_peer_view_update(const libp2p::peer::PeerId &peer_id, + const network::View &view); + + void send_peer_messages_for_relay_parent( + const libp2p::peer::PeerId &peer_id, const RelayHash &relay_parent); + + /** + * @brief Sends peer messages corresponding for a given relay parent. + * + * @param peer_id Optional reference to the PeerId of the peer to send the + * messages to. + * @param relay_parent The hash of the relay parent block + */ + std::optional, + network::VersionedValidatorProtocolMessage>> + pending_statement_network_message( + const StatementStore &statement_store, + const RelayHash &relay_parent, + const libp2p::peer::PeerId &peer, + network::CollationVersion version, + ValidatorIndex originator, + const network::vstaging::CompactStatement &compact); + + void send_cluster_candidate_statements(const CandidateHash &candidate_hash, + const RelayHash &relay_parent); + + void apply_post_confirmation(const PostConfirmation &post_confirmation); + + void send_pending_cluster_statements( + const RelayHash &relay_parent, + const libp2p::peer::PeerId &peer_id, + network::CollationVersion version, + ValidatorIndex peer_validator_id, + PerRelayParentState &relay_parent_state); + + void send_pending_grid_messages(const RelayHash &relay_parent, + const libp2p::peer::PeerId &peer_id, + network::CollationVersion version, + ValidatorIndex peer_validator_id, + const Groups &groups, + PerRelayParentState &relay_parent_state); + + /** + * @brief Circulates a statement to the validators group. + * @param relay_parent The hash of the relay parent block. This is used to + * identify the group of validators to which the statement should be sent. + * @param statement The statement to be circulated. This is an indexed and + * signed compact statement. + */ + void circulate_statement( + const RelayHash &relay_parent, + PerRelayParentState &relay_parent_state, + const IndexedAndSigned &statement); + + outcome::result< + std::reference_wrapper> + check_statement_signature( + SessionIndex session_index, + const std::vector &validators, + const RelayHash &relay_parent, + const network::vstaging::SignedCompactStatement &statement); + + /// Checks whether a statement is allowed, whether the signature is + /// accurate, + /// and importing into the cluster tracker if successful. + /// + /// if successful, this returns a checked signed statement if it should be + /// imported or otherwise an error indicating a reputational fault. + outcome::result> + handle_cluster_statement( + const RelayHash &relay_parent, + ClusterTracker &cluster_tracker, + SessionIndex session, + const runtime::SessionInfo &session_info, + const network::vstaging::SignedCompactStatement &statement, + ValidatorIndex cluster_sender_index); + + outcome::result handle_grid_statement( + const RelayHash &relay_parent, + PerRelayParentState &per_relay_parent, + grid::GridTracker &grid_tracker, + const IndexedAndSigned &statement, + ValidatorIndex grid_sender_index); + + void send_backing_fresh_statements(const ConfirmedCandidate &confirmed, + const RelayHash &relay_parent, + PerRelayParentState &per_relay_parent, + const std::vector &group, + const CandidateHash &candidate_hash); + + network::vstaging::StatementFilter local_knowledge_filter( + size_t group_size, + GroupIndex group_index, + const CandidateHash &candidate_hash, + const StatementStore &statement_store); + + void provide_candidate_to_grid( + const CandidateHash &candidate_hash, + PerRelayParentState &relay_parent_state, + const ConfirmedCandidate &confirmed_candidate, + const runtime::SessionInfo &session_info); + + void fragment_chain_update_inner( + std::optional> active_leaf_hash, + std::optional, + ParachainId>> required_parent_info, + std::optional> + known_hypotheticals); + + void new_confirmed_candidate_fragment_chain_updates( + const HypotheticalCandidate &candidate); + + void new_leaf_fragment_chain_updates(const Hash &leaf_hash); + + void prospective_backed_notification_fragment_chain_updates( + ParachainId para_id, const Hash ¶_head); + + ManifestImportSuccessOpt handle_incoming_manifest_common( + const libp2p::peer::PeerId &peer_id, + const CandidateHash &candidate_hash, + const RelayHash &relay_parent, + ManifestSummary manifest_summary, + ParachainId para_id, + grid::ManifestKind manifest_kind); + + std::deque + post_acknowledgement_statement_messages( + ValidatorIndex recipient, + const RelayHash &relay_parent, + grid::GridTracker &grid_tracker, + const StatementStore &statement_store, + const Groups &groups, + GroupIndex group_index, + const CandidateHash &candidate_hash, + const libp2p::peer::PeerId &peer, + network::CollationVersion version); + + std::deque, + network::VersionedValidatorProtocolMessage>> + acknowledgement_and_statement_messages( + const libp2p::peer::PeerId &peer, + network::CollationVersion version, + ValidatorIndex validator_index, + const Groups &groups, + PerRelayParentState &relay_parent_state, + const RelayHash &relay_parent, + GroupIndex group_index, + const CandidateHash &candidate_hash, + const network::vstaging::StatementFilter &local_knowledge); + + void request_hypotetical_membership( + std::vector hypotheticals, + std::optional active_leaf); + + void process_frontier( + std::vector> frontier); + + outcome::result> is_parachain_validator( + const primitives::BlockHash &relay_parent) const; + + outcome::result> + fetch_claim_queue(const RelayHash &relay_parent); + + std::unordered_map> + determine_groups_per_para( + const std::vector &availability_cores, + const runtime::GroupDescriptor &group_rotation_info, + std::optional &maybe_claim_queue, + size_t max_candidate_depth); + + outcome::result handle_view_event(const network::ExView &event); + void handle_active_leaves_update( + const network::ExView &event, + std::vector new_contexts); + outcome::result handle_active_leaves_update_inner( + const network::ExView &event, + std::vector new_contexts); + outcome::result handle_deactive_leaves_update_inner( + const std::vector &lost); + outcome::result update_our_view(const Hash &relay_parent, + const network::View &view); + + void on_peer_connected(const libp2p::peer::PeerId &peer); + void on_peer_disconnected(const libp2p::peer::PeerId &peer); + + private: + log::Logger logger = + log::createLogger("StatementDistribution", "parachain"); + + SafeObject implicit_view; + Candidates candidates; + std::unordered_map + per_relay_parent; + std::shared_ptr> per_session; + std::unordered_map peers; + std::shared_ptr signer_factory; + std::shared_ptr peer_use_count; + + /// worker thread + std::shared_ptr statements_distribution_thread_handler; + std::shared_ptr query_audi; + std::weak_ptr parachain_processor; + std::shared_ptr network_bridge; + std::shared_ptr router; + std::shared_ptr main_pool_handler; + std::shared_ptr hasher; + std::shared_ptr prospective_parachains; + std::shared_ptr parachain_host; + std::shared_ptr crypto_provider; + std::shared_ptr peer_view; + std::shared_ptr block_tree; + LazySPtr slots_util; + std::shared_ptr babe_config_repo; + + /// sub + primitives::events::PeerEventSubscriberPtr peer_state_sub; + network::PeerView::MyViewSubscriberPtr my_view_sub; + network::PeerView::PeerViewSubscriberPtr remote_view_sub; + }; + +} // namespace kagome::parachain::statement_distribution + +OUTCOME_HPP_DECLARE_ERROR(kagome::parachain::statement_distribution, + StatementDistribution::Error); diff --git a/core/parachain/validator/statement_distribution/types.hpp b/core/parachain/validator/statement_distribution/types.hpp new file mode 100644 index 0000000000..fb20cf0367 --- /dev/null +++ b/core/parachain/validator/statement_distribution/types.hpp @@ -0,0 +1,72 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "network/types/collator_messages_vstaging.hpp" +#include "runtime/runtime_api/parachain_host_types.hpp" + +namespace kagome::parachain { + + struct StatementWithPVDSeconded { + network::CommittedCandidateReceipt committed_receipt; + runtime::PersistedValidationData pvd; + }; + + struct StatementWithPVDValid { + parachain::CandidateHash candidate_hash; + }; + + using StatementWithPVD = + boost::variant; + + using SignedFullStatementWithPVD = + parachain::IndexedAndSigned; + + inline primitives::BlockHash candidateHashFrom( + const StatementWithPVD &statement, + const std::shared_ptr &hasher) { + return visit_in_place( + statement, + [&](const StatementWithPVDSeconded &val) { + return hasher->blake2b_256( + ::scale::encode(val.committed_receipt.to_plain(*hasher)).value()); + }, + [&](const StatementWithPVDValid &val) { return val.candidate_hash; }); + } + + /** + * @brief Converts a SignedFullStatementWithPVD to an IndexedAndSigned + * CompactStatement. + */ + inline IndexedAndSigned + signed_to_compact(const SignedFullStatementWithPVD &s, + const std::shared_ptr &hasher) { + const Hash h = candidateHashFrom(getPayload(s), hasher); + return { + .payload = + { + .payload = visit_in_place( + getPayload(s), + [&](const StatementWithPVDSeconded &) + -> network::vstaging::CompactStatement { + return network::vstaging::SecondedCandidateHash{ + .hash = h, + }; + }, + [&](const StatementWithPVDValid &) + -> network::vstaging::CompactStatement { + return network::vstaging::ValidCandidateHash{ + .hash = h, + }; + }), + .ix = s.payload.ix, + }, + .signature = s.signature, + }; + } + +} // namespace kagome::parachain diff --git a/core/primitives/event_types.hpp b/core/primitives/event_types.hpp index 5fb451ef15..03e7422668 100644 --- a/core/primitives/event_types.hpp +++ b/core/primitives/event_types.hpp @@ -43,6 +43,11 @@ namespace kagome::primitives::events { kDeactivateAfterFinalization = 6, }; + enum struct PeerEventType : uint8_t { + kConnected = 1, + kDisconnected = 2, + }; + enum struct SyncStateEventType : uint8_t { kSyncState = 1 }; using HeadsEventParams = ref_t; @@ -233,16 +238,22 @@ namespace kagome::primitives::events { primitives::BlockHash>; using StorageSubscriptionEnginePtr = std::shared_ptr; - using StorageEventSubscriber = StorageSubscriptionEngine::SubscriberType; using StorageEventSubscriberPtr = std::shared_ptr; + using PeerSubscriptionEngine = + subscription::SubscriptionEngine; + using PeerSubscriptionEnginePtr = std::shared_ptr; + using PeerEventSubscriber = PeerSubscriptionEngine::SubscriberType; + using PeerEventSubscriberPtr = std::shared_ptr; + using ChainSubscriptionEngine = subscription::SubscriptionEngine, primitives::events::ChainEventParams>; using ChainSubscriptionEnginePtr = std::shared_ptr; - using ChainEventSubscriber = ChainSubscriptionEngine::SubscriberType; using ChainEventSubscriberPtr = std::shared_ptr; @@ -252,7 +263,6 @@ namespace kagome::primitives::events { primitives::events::SyncStateEventParams>; using SyncStateSubscriptionEnginePtr = std::shared_ptr; - using SyncStateEventSubscriber = SyncStateSubscriptionEngine::SubscriberType; using SyncStateEventSubscriberPtr = std::shared_ptr; @@ -262,7 +272,6 @@ namespace kagome::primitives::events { primitives::events::ExtrinsicLifecycleEvent>; using ExtrinsicSubscriptionEnginePtr = std::shared_ptr; - using ExtrinsicEventSubscriber = ExtrinsicSubscriptionEngine::SubscriberType; using ExtrinsicEventSubscriberPtr = std::shared_ptr; @@ -282,7 +291,7 @@ namespace kagome::primitives::events { struct ChainSub { ChainSub(ChainSubscriptionEnginePtr engine) : sub{std::make_shared( - std::move(engine))} {} + std::move(engine))} {} void onBlock(ChainEventType type, auto f) { subscribe(*sub, type, [f{std::move(f)}](const ChainEventParams &args) { diff --git a/core/utils/map.hpp b/core/utils/map.hpp index aa2c9f31b6..bd4444004d 100644 --- a/core/utils/map.hpp +++ b/core/utils/map.hpp @@ -87,6 +87,16 @@ namespace kagome::utils { return val; } + template + requires requires { typename C::mapped_type; } + inline std::optional get_it( + C &container, const typename C::key_type &key) { + if (auto it = container.find(key); it != container.end()) { + return it; + } + return std::nullopt; + } + } // namespace kagome::utils #endif // KAGOME_UTILS_MAP_HPP diff --git a/core/utils/retain_if.hpp b/core/utils/retain_if.hpp index bf49ba0fe2..1813388572 100644 --- a/core/utils/retain_if.hpp +++ b/core/utils/retain_if.hpp @@ -17,8 +17,9 @@ namespace kagome { v.begin(), v.end(), [&](T &v) { return not predicate(v); }), v.end()); } - template - void retain_if(std::unordered_set &v, auto &&predicate) { + template + requires requires { typename C::key_type; } + void retain_if(C &v, auto &&predicate) { for (auto it = v.begin(); it != v.end();) { if (!predicate(*it)) { it = v.erase(it); @@ -27,4 +28,5 @@ namespace kagome { } } } + } // namespace kagome diff --git a/test/core/parachain/grid_tracker.cpp b/test/core/parachain/grid_tracker.cpp index 6459cbea89..fbad8e3809 100644 --- a/test/core/parachain/grid_tracker.cpp +++ b/test/core/parachain/grid_tracker.cpp @@ -898,3 +898,108 @@ TEST_F(GridTrackerTest, session_grid_topology_consistent) { } } } + +TEST_F(GridTrackerTest, knowledge_rejects_conflicting_manifest) { + ReceivedManifests knowledge; + + const ManifestSummary expected_manifest_summary{ + .claimed_parent_hash = fromNumber(2), + .claimed_group_index = GroupIndex(0), + .statement_knowledge = + create_filter({{true, true, false}, {false, true, true}}), + }; + + ASSERT_TRUE( + knowledge.import_received(3, 2, fromNumber(1), expected_manifest_summary) + .has_value()); + + // conflicting group + { + auto s = expected_manifest_summary; + s.claimed_group_index = GroupIndex(1); + EXPECT_EC(knowledge.import_received(3, 2, fromNumber(1), s), + GridTracker::Error::CONFLICTING); + } + + // conflicting parent hash + { + auto s = expected_manifest_summary; + s.claimed_parent_hash = fromNumber(3); + EXPECT_EC(knowledge.import_received(3, 2, fromNumber(1), s), + GridTracker::Error::CONFLICTING); + } + + // conflicting seconded statements bitfield + { + auto s = expected_manifest_summary; + s.statement_knowledge.seconded_in_group.bits = {false, true, false}; + EXPECT_EC(knowledge.import_received(3, 2, fromNumber(1), s), + GridTracker::Error::CONFLICTING); + } + + // conflicting valid statements bitfield + { + auto s = expected_manifest_summary; + s.statement_knowledge.validated_in_group.bits = {false, true, false}; + EXPECT_EC(knowledge.import_received(3, 2, fromNumber(1), s), + GridTracker::Error::CONFLICTING); + } +} + +TEST_F(GridTrackerTest, reject_overflowing_manifests) { + ReceivedManifests knowledge; + + ASSERT_TRUE( + knowledge + .import_received(3, + 2, + fromNumber(1), + ManifestSummary{ + .claimed_parent_hash = fromNumber(0xA), + .claimed_group_index = GroupIndex(0), + .statement_knowledge = create_filter( + {{true, true, false}, {false, true, true}}), + }) + .has_value()); + + ASSERT_TRUE( + knowledge + .import_received(3, + 2, + fromNumber(2), + ManifestSummary{ + .claimed_parent_hash = fromNumber(0xB), + .claimed_group_index = GroupIndex(0), + .statement_knowledge = create_filter( + {{true, false, true}, {false, true, true}}), + }) + .has_value()); + + // Reject a seconding validator that is already at the seconding limit. + // Seconding counts for the validators should not be applied. + EXPECT_EC(knowledge.import_received( + 3, + 2, + fromNumber(3), + ManifestSummary{ + .claimed_parent_hash = fromNumber(0xC), + .claimed_group_index = GroupIndex(0), + .statement_knowledge = create_filter( + {{true, true, true}, {false, true, true}}), + }), + GridTracker::Error::SECONDING_OVERFLOW); + + // Don't reject validators that have seconded less than the limit so far. + ASSERT_TRUE( + knowledge + .import_received(3, + 2, + fromNumber(3), + ManifestSummary{ + .claimed_parent_hash = fromNumber(0xC), + .claimed_group_index = GroupIndex(0), + .statement_knowledge = create_filter( + {{false, true, true}, {false, true, true}}), + }) + .has_value()); +}