Validation v3 (#2130)

* validation v3 Signed-off-by: iceseer <[email protected]> * disputes fixup Signed-off-by: iceseer <[email protected]> -------- Signed-off-by: iceseer <[email protected]> Co-authored-by: kamilsa <[email protected]>
qdrvm · Aug 2, 2024 · 95201a8 · 95201a8
1 parent a0aee09
commit 95201a8
Show file tree

Hide file tree

Showing 20 changed files with 1,474 additions and 575 deletions.
diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake
@@ -1,5 +1,5 @@
 HunterGate(
-  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm18.zip
-  SHA1 22d842b448f84a39392d7835f4046da34f8dcd70
+  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm19.zip
+  SHA1 eed8b8333c14f25176d4af4fb26256981fd1b527
   LOCAL
-)
+)
diff --git a/core/api/transport/impl/ws/ws_session.cpp b/core/api/transport/impl/ws/ws_session.cpp
@@ -10,6 +10,7 @@
 
 #include <boost/asio/dispatch.hpp>
 #include <boost/beast/core/bind_handler.hpp>
+#include <boost/beast/http/string_body.hpp>
 #include <boost/config.hpp>
 #include <libp2p/outcome/outcome.hpp>
 

diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp
@@ -387,6 +387,8 @@ namespace kagome::blockchain {
       block_tree_leaves.erase(block);
 
       std::vector<primitives::BlockHash> leaves;
+      leaves.reserve(block_tree_leaves.size());
+
       std::transform(block_tree_leaves.begin(),
                      block_tree_leaves.end(),
                      std::back_inserter(leaves),
@@ -423,18 +425,18 @@ namespace kagome::blockchain {
       std::shared_ptr<storage::trie_pruner::TriePruner> state_pruner,
       common::MainThreadPool &main_thread_pool)
       : block_tree_data_{BlockTreeData{
-          .header_repo_ = std::move(header_repo),
-          .storage_ = std::move(storage),
-          .state_pruner_ = std::move(state_pruner),
-          .tree_ = std::make_unique<CachedTree>(finalized),
-          .extrinsic_observer_ = std::move(extrinsic_observer),
-          .hasher_ = std::move(hasher),
-          .extrinsic_event_key_repo_ = std::move(extrinsic_event_key_repo),
-          .justification_storage_policy_ =
-              std::move(justification_storage_policy),
-          .genesis_block_hash_ = {},
-          .blocks_pruning_ = {app_config.blocksPruning(), finalized.number},
-      }},
+            .header_repo_ = std::move(header_repo),
+            .storage_ = std::move(storage),
+            .state_pruner_ = std::move(state_pruner),
+            .tree_ = std::make_unique<CachedTree>(finalized),
+            .extrinsic_observer_ = std::move(extrinsic_observer),
+            .hasher_ = std::move(hasher),
+            .extrinsic_event_key_repo_ = std::move(extrinsic_event_key_repo),
+            .justification_storage_policy_ =
+                std::move(justification_storage_policy),
+            .genesis_block_hash_ = {},
+            .blocks_pruning_ = {app_config.blocksPruning(), finalized.number},
+        }},
         main_pool_handler_{main_thread_pool.handlerStarted()} {
     block_tree_data_.sharedAccess([&](const BlockTreeData &p) {
       BOOST_ASSERT(p.header_repo_ != nullptr);

diff --git a/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp b/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp
@@ -1205,25 +1205,25 @@ namespace kagome::dispute {
     auto is_old_concluded_for =
         intermediate_result.old_state.dispute_status.has_value()
             ? is_type<ConcludedFor>(
-                intermediate_result.old_state.dispute_status.value())
+                  intermediate_result.old_state.dispute_status.value())
             : false;
     auto is_new_concluded_for =
         intermediate_result.new_state.dispute_status.has_value()
             ? is_type<ConcludedFor>(
-                intermediate_result.new_state.dispute_status.value())
+                  intermediate_result.new_state.dispute_status.value())
             : false;
     auto is_freshly_concluded_for =
         not is_old_concluded_for and is_new_concluded_for;
 
     auto is_old_concluded_against =
         intermediate_result.old_state.dispute_status.has_value()
             ? is_type<ConcludedAgainst>(
-                intermediate_result.old_state.dispute_status.value())
+                  intermediate_result.old_state.dispute_status.value())
             : false;
     auto is_new_concluded_against =
         intermediate_result.new_state.dispute_status.has_value()
             ? is_type<ConcludedAgainst>(
-                intermediate_result.new_state.dispute_status.value())
+                  intermediate_result.new_state.dispute_status.value())
             : false;
     auto is_freshly_concluded_against =
         not is_old_concluded_against and is_new_concluded_against;
@@ -1234,12 +1234,12 @@ namespace kagome::dispute {
     auto is_old_confirmed_concluded =
         intermediate_result.old_state.dispute_status.has_value()
             ? not is_type<Active>(
-                intermediate_result.old_state.dispute_status.value())
+                  intermediate_result.old_state.dispute_status.value())
             : false;
     auto is_new_confirmed_concluded =
         intermediate_result.new_state.dispute_status.has_value()
             ? not is_type<Active>(
-                intermediate_result.new_state.dispute_status.value())
+                  intermediate_result.new_state.dispute_status.value())
             : false;
     auto is_freshly_confirmed =
         not is_old_confirmed_concluded and is_new_confirmed_concluded;
@@ -1262,13 +1262,13 @@ namespace kagome::dispute {
       // {polkadot}/node/core/dispute-coordinator/src/initialized.rs:809
 
       auto promise_res = std::promise<
-          std::unordered_map<ValidatorIndex, ValidatorSignature>>();
+          parachain::ApprovalDistribution::SignaturesForCandidate>();
       auto res_future = promise_res.get_future();
 
       approval_distribution_->getApprovalSignaturesForCandidate(
           candidate_hash,
           [promise_res = std::ref(promise_res)](
-              std::unordered_map<ValidatorIndex, ValidatorSignature> res) {
+              parachain::ApprovalDistribution::SignaturesForCandidate res) {
             promise_res.get().set_value(std::move(res));
           });
 
@@ -1289,7 +1289,8 @@ namespace kagome::dispute {
 
         auto _votes = std::move(import_result.new_state.votes);
 
-        for (auto &[index, signature] : approval_votes) {
+        for (auto &[index, signatures_data] : approval_votes) {
+          auto &[hash, candidates, signature] = signatures_data;
           // clang-format off
           BOOST_ASSERT_MSG(
               [&] {

diff --git a/core/network/common.hpp b/core/network/common.hpp
@@ -32,7 +32,7 @@ namespace kagome::network {
   const libp2p::peer::ProtocolName kCollationProtocolVStaging{
       "/{}/collation/2"};
   const libp2p::peer::ProtocolName kValidationProtocolVStaging{
-      "/{}/validation/2"};
+      "/{}/validation/3"};
   const libp2p::peer::ProtocolName kReqCollationProtocol{"/{}/req_collation/1"};
   const libp2p::peer::ProtocolName kReqCollationVStagingProtocol{
       "/{}/req_collation/2"};

diff --git a/core/network/impl/peer_view.cpp b/core/network/impl/peer_view.cpp
@@ -77,22 +77,42 @@ namespace kagome::network {
     }
   }
 
+  size_t PeerView::peersCount() const {
+    return remote_view_.sharedAccess([](const auto &rv) { return rv.size(); });
+  }
+
   void PeerView::removePeer(const PeerId &peer_id) {
-    if (auto it = remote_view_.find(peer_id); it != remote_view_.end()) {
-      network::View old_view{std::move(it->second)};
-      remote_view_.erase(peer_id);
+    auto ref = remote_view_.exclusiveAccess(
+        [&](auto &rv) -> std::optional<network::View> {
+          if (auto it = rv.find(peer_id); it != rv.end()) {
+            network::View old_view{std::move(it->second)};
+            rv.erase(peer_id);
+            return old_view;
+          }
+          return std::nullopt;
+        });
+
+    if (ref) {
       remote_view_update_observable_->notify(
-          EventType::kPeerRemoved, peer_id, std::move(old_view));
+          EventType::kPeerRemoved, peer_id, std::move(*ref));
     }
   }
 
   void PeerView::updateRemoteView(const PeerId &peer_id, network::View &&view) {
-    auto it = remote_view_.find(peer_id);
-    if (it == remote_view_.end() || it->second != view) {
-      auto &ref = remote_view_[peer_id];
-      ref = std::move(view);
+    auto ref = remote_view_.exclusiveAccess(
+        [&](auto &rv) -> std::optional<network::View> {
+          auto it = rv.find(peer_id);
+          if (it == rv.end() || it->second != view) {
+            auto &ref = rv[peer_id];
+            ref = std::move(view);
+            return ref;
+          }
+          return std::nullopt;
+        });
+
+    if (ref) {
       remote_view_update_observable_->notify(
-          EventType::kViewUpdated, peer_id, ref);
+          EventType::kViewUpdated, peer_id, *ref);
     }
   }
 

diff --git a/core/network/peer_view.hpp b/core/network/peer_view.hpp
@@ -20,6 +20,7 @@
 #include "subscription/subscriber.hpp"
 #include "subscription/subscription_engine.hpp"
 #include "utils/non_copyable.hpp"
+#include "utils/safe_object.hpp"
 
 namespace kagome::blockchain {
   class BlockTree;
@@ -75,6 +76,8 @@ namespace kagome::network {
     bool prepare();
     void stop();
 
+    size_t peersCount() const;
+
     MyViewSubscriptionEnginePtr getMyViewObservable();
     PeerViewSubscriptionEnginePtr getRemoteViewObservable();
 
@@ -91,7 +94,7 @@ namespace kagome::network {
     PeerViewSubscriptionEnginePtr remote_view_update_observable_;
 
     std::optional<ExView> my_view_;
-    std::unordered_map<PeerId, View> remote_view_;
+    SafeObject<std::unordered_map<PeerId, View>> remote_view_;
     LazySPtr<blockchain::BlockTree> block_tree_;
   };
 

diff --git a/core/network/types/collator_messages.hpp b/core/network/types/collator_messages.hpp
@@ -358,21 +358,8 @@ namespace kagome::network {
    */
   using BitfieldDistributionMessage = boost::variant<BitfieldDistribution>;
 
-  /// A signed approval vote which references the candidate indirectly via the
-  /// block.
-  ///
-  /// In practice, we have a look-up from block hash and candidate index to
-  /// candidate hash, so this can be transformed into a `SignedApprovalVote`.
-  struct ApprovalVote {
-    SCALE_TIE(2);
-
-    primitives::BlockHash
-        block_hash;  /// A block hash where the candidate appears.
-    CandidateIndex
-        candidate_index;  /// The index of the candidate in the list of
-                          /// candidates fully included as-of the block.
-  };
-  using IndirectSignedApprovalVote = parachain::IndexedAndSigned<ApprovalVote>;
+  using IndirectSignedApprovalVote =
+      parachain::approval::IndirectSignedApprovalVote;
 
   struct Assignment {
     SCALE_TIE(2);

diff --git a/core/network/types/collator_messages_vstaging.hpp b/core/network/types/collator_messages_vstaging.hpp
@@ -35,9 +35,43 @@ namespace kagome::network::vstaging {
   using CollatorProtocolMessageCollationSeconded = network::Seconded;
   using BitfieldDistributionMessage = network::BitfieldDistributionMessage;
   using BitfieldDistribution = network::BitfieldDistribution;
-  using ApprovalDistributionMessage = network::ApprovalDistributionMessage;
   using ViewUpdate = network::ViewUpdate;
 
+  using IndirectSignedApprovalVoteV2 =
+      parachain::approval::IndirectSignedApprovalVoteV2;
+
+  struct Assignment {
+    SCALE_TIE(2);
+
+    kagome::parachain::approval::IndirectAssignmentCertV2
+        indirect_assignment_cert;
+    scale::BitVec candidate_bitfield;
+  };
+
+  struct Assignments {
+    SCALE_TIE(1);
+
+    std::vector<Assignment> assignments;  /// Assignments for candidates in
+                                          /// recent, unfinalized blocks.
+  };
+
+  struct Approvals {
+    SCALE_TIE(1);
+
+    std::vector<IndirectSignedApprovalVoteV2>
+        approvals;  /// Approvals for candidates in some recent, unfinalized
+                    /// block.
+  };
+
+  /// Network messages used by the approval distribution subsystem.
+  using ApprovalDistributionMessage = boost::variant<
+      /// Assignments for candidates in recent, unfinalized blocks.
+      ///
+      /// Actually checking the assignment may yield a different result.
+      Assignments,
+      /// Approvals for candidates in some recent, unfinalized block.
+      Approvals>;
+
   struct CollatorProtocolMessageAdvertiseCollation {
     SCALE_TIE(3);
     /// Hash of the relay parent advertised collation is based on.

diff --git a/core/parachain/approval/approval.hpp b/core/parachain/approval/approval.hpp
@@ -80,6 +80,25 @@ namespace kagome::parachain::approval {
 
     /// The VRF showing the criterion is met.
     crypto::VRFOutput vrf;
+
+    static AssignmentCertV2 from(const AssignmentCert &src) {
+      return AssignmentCertV2{
+          .kind = visit_in_place(
+              src.kind,
+              [](const auto &v) -> AssignmentCertKindV2 { return v; }),
+          .vrf = src.vrf,
+      };
+    }
+  };
+
+  /// An assignment criterion which refers to the candidate under which the
+  /// assignment is relevant by block hash.
+  struct IndirectAssignmentCert {
+    SCALE_TIE(3);
+
+    Hash block_hash;           /// A block hash where the candidate appears.
+    ValidatorIndex validator;  /// The validator index.
+    AssignmentCert cert;       /// The cert itself.
   };
 
   /// An assignment criterion which refers to the candidate under which the
@@ -95,17 +114,30 @@ namespace kagome::parachain::approval {
 
     /// The cert itself.
     AssignmentCertV2 cert;
+
+    static IndirectAssignmentCertV2 from(const IndirectAssignmentCert &src) {
+      return IndirectAssignmentCertV2{
+          .block_hash = src.block_hash,
+          .validator = src.validator,
+          .cert = AssignmentCertV2::from(src.cert),
+      };
+    }
   };
 
-  /// An assignment criterion which refers to the candidate under which the
-  /// assignment is relevant by block hash.
-  struct IndirectAssignmentCert {
-    SCALE_TIE(3);
+  /// A signed approval vote which references the candidate indirectly via the
+  /// block.
+  ///
+  /// In practice, we have a look-up from block hash and candidate index to
+  /// candidate hash, so this can be transformed into a `SignedApprovalVote`.
+  struct IndirectApprovalVote {
+    SCALE_TIE(2);
 
-    Hash block_hash;           /// A block hash where the candidate appears.
-    ValidatorIndex validator;  /// The validator index.
-    AssignmentCert cert;       /// The cert itself.
+    Hash block_hash;  /// A block hash where the candidate appears.
+    CandidateIndex
+        candidate_index;  /// The index of the candidate in the list of
+                          /// candidates fully included as-of the block.
   };
+  using IndirectSignedApprovalVote = IndexedAndSigned<IndirectApprovalVote>;
 
   /// A signed approval vote which references the candidate indirectly via the
   /// block.
@@ -121,23 +153,30 @@ namespace kagome::parachain::approval {
     /// The index of the candidate in the list of candidates fully included
     /// as-of the block.
     scale::BitVec candidate_indices;
+
+    static IndirectApprovalVoteV2 from(const IndirectApprovalVote &value) {
+      scale::BitVec v;
+      v.bits.resize(value.candidate_index + 1);
+      v.bits[value.candidate_index] = true;
+      return IndirectApprovalVoteV2{
+          .block_hash = value.block_hash,
+          .candidate_indices = std::move(v),
+      };
+    }
   };
   using IndirectSignedApprovalVoteV2 = IndexedAndSigned<IndirectApprovalVoteV2>;
 
-  /// A signed approval vote which references the candidate indirectly via the
-  /// block.
-  ///
-  /// In practice, we have a look-up from block hash and candidate index to
-  /// candidate hash, so this can be transformed into a `SignedApprovalVote`.
-  struct IndirectApprovalVote {
-    SCALE_TIE(2);
-
-    Hash block_hash;  /// A block hash where the candidate appears.
-    CandidateIndex
-        candidate_index;  /// The index of the candidate in the list of
-                          /// candidates fully included as-of the block.
-  };
-  using IndirectSignedApprovalVote = IndexedAndSigned<IndirectApprovalVote>;
+  inline IndirectSignedApprovalVoteV2 from(
+      const IndirectSignedApprovalVote &value) {
+    return IndirectSignedApprovalVoteV2{
+        .payload =
+            {
+                .payload = IndirectApprovalVoteV2::from(value.payload.payload),
+                .ix = value.payload.ix,
+            },
+        .signature = value.signature,
+    };
+  }
 
   struct RemoteApproval {
     ValidatorIndex validator_ix;