Merge remote-tracking branch 'origin/master' into feature/prospective…

…_parachains_multi_candidates Signed-off-by: iceseer <[email protected]> # Conflicts: # core/parachain/pvf/pvf_impl.cpp # core/parachain/pvf/pvf_impl.hpp # core/runtime/common/runtime_instances_pool.cpp # test/core/parachain/pvf_test.cpp
qdrvm · Apr 1, 2024 · 70cfba1 · 70cfba1
2 parents 4fecba8 + e22c53e
commit 70cfba1
Show file tree

Hide file tree

Showing 156 changed files with 2,619 additions and 744 deletions.
diff --git a/.githooks/README.md b/.githooks/README.md
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -86,8 +86,8 @@ jobs:
         options:
           - name: "Self-hosted: Linux: gcc-12 ASAN"
             run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON
-          - name: "Self-hosted: Linux: clang-15 TSAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DTSAN=ON
+          - name: "Self-hosted: Linux: clang-15 TSAN WAVM"
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM
           - name: "Self-hosted: Linux: clang-15 UBSAN"
             run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DUBSAN=ON
           - name: "Self-hosted: Linux: clang-15 External Project"
@@ -124,8 +124,8 @@ jobs:
         options:
           - name: "Self-hosted: Linux: gcc-12 ASAN"
             run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON
-          - name: "Self-hosted: Linux: clang-15 TSAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DTSAN=ON
+          - name: "Self-hosted: Linux: clang-15 TSAN WAVM"
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM
           - name: "Self-hosted: Linux: clang-15 UBSAN"
             run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DUBSAN=ON
           - name: "Self-hosted: Linux: clang-15 External Project"

diff --git a/cmake/toolchain/compiler/gcc-13.cmake b/cmake/toolchain/compiler/gcc-13.cmake
@@ -8,11 +8,11 @@ find_program(CMAKE_C_COMPILER gcc-13)
 find_program(CMAKE_CXX_COMPILER g++-13)
 
 if(NOT CMAKE_C_COMPILER)
-  fatal_error("gcc-13 not found")
+  message(FATAL_ERROR "gcc-13 not found")
 endif()
 
 if(NOT CMAKE_CXX_COMPILER)
-  fatal_error("g++-13 not found")
+message(FATAL_ERROR "g++-13 not found")
 endif()
 
 set(

diff --git a/core/api/service/author/author_api.hpp b/core/api/service/author/author_api.hpp
@@ -9,6 +9,7 @@
 #include "common/blob.hpp"
 #include "common/buffer.hpp"
 #include "common/buffer_view.hpp"
+#include "crypto/common.hpp"
 #include "crypto/crypto_store/key_type.hpp"
 #include "primitives/author_api_primitives.hpp"
 #include "primitives/transaction_validity.hpp"
@@ -53,7 +54,7 @@ namespace kagome::api {
      * @param public_key The public key in binary
      */
     virtual outcome::result<void> insertKey(crypto::KeyType key_type,
-                                            const BufferView &seed,
+                                            crypto::SecureBuffer<> seed,
                                             const BufferView &public_key) = 0;
 
     /**

diff --git a/core/api/service/author/impl/author_api_impl.cpp b/core/api/service/author/impl/author_api_impl.cpp
@@ -62,7 +62,7 @@ namespace kagome::api {
   }
 
   outcome::result<void> AuthorApiImpl::insertKey(crypto::KeyType key_type_id,
-                                                 const BufferView &seed,
+                                                 crypto::SecureBuffer<> seed,
                                                  const BufferView &public_key) {
     if (std::find(kKeyTypes.begin(), kKeyTypes.end(), key_type_id)
         == kKeyTypes.end()) {
@@ -77,7 +77,7 @@ namespace kagome::api {
     };
     if (crypto::KeyTypes::BABE == key_type_id
         or crypto::KeyTypes::AUTHORITY_DISCOVERY == key_type_id) {
-      OUTCOME_TRY(seed_typed, crypto::Sr25519Seed::fromSpan(seed));
+      OUTCOME_TRY(seed_typed, crypto::Sr25519Seed::from(seed));
       OUTCOME_TRY(public_key_typed,
                   crypto::Sr25519PublicKey::fromSpan(public_key));
       OUTCOME_TRY(keypair,
@@ -87,7 +87,7 @@ namespace kagome::api {
       }
     }
     if (crypto::KeyTypes::GRANDPA == key_type_id) {
-      OUTCOME_TRY(seed_typed, crypto::Ed25519Seed::fromSpan(seed));
+      OUTCOME_TRY(seed_typed, crypto::Ed25519Seed::from(seed));
       OUTCOME_TRY(public_key_typed,
                   crypto::Ed25519PublicKey::fromSpan(public_key));
       OUTCOME_TRY(keypair,
@@ -97,7 +97,8 @@ namespace kagome::api {
         return outcome::failure(crypto::CryptoStoreError::WRONG_PUBLIC_KEY);
       }
     }
-    auto res = key_store_->saveKeyPair(key_type_id, public_key, seed);
+    auto res =
+        key_store_->saveKeyPair(key_type_id, public_key, std::move(seed));
     return res;
   }
 

diff --git a/core/api/service/author/impl/author_api_impl.hpp b/core/api/service/author/impl/author_api_impl.hpp
@@ -83,7 +83,7 @@ namespace kagome::api {
         const primitives::Extrinsic &extrinsic) override;
 
     outcome::result<void> insertKey(crypto::KeyType key_type,
-                                    const BufferView &seed,
+                                    crypto::SecureBuffer<> seed,
                                     const BufferView &public_key) override;
 
     outcome::result<common::Buffer> rotateKeys() override;

diff --git a/core/api/service/author/requests/insert_key.hpp b/core/api/service/author/requests/insert_key.hpp
@@ -10,23 +10,32 @@
 
 #include "api/service/author/author_api.hpp"
 #include "api/service/base_request.hpp"
+#include "crypto/common.hpp"
 #include "outcome/outcome.hpp"
 
 namespace kagome::api::author::request {
 
   class InsertKey final
-      : public details::
-            RequestType<void, std::string, std::string, std::string> {
+      : public details::RequestType<
+            void,
+            std::string,
+            std::vector<char, crypto::SecureHeapAllocator<char>>,
+            std::string> {
    public:
     explicit InsertKey(std::shared_ptr<AuthorApi> api) : api_(std::move(api)) {
       BOOST_ASSERT(api_);
     };
 
     outcome::result<Return> execute() override {
-      OUTCOME_TRY(seed, common::unhexWith0x(getParam<1>()));
+      auto &seed_hex = getParam<1>();
+      crypto::SecureBuffer<> seed_buf(seed_hex.size() / 2 - 1);
+      OUTCOME_TRY(common::unhexWith0x(
+          std::string_view{seed_hex.data(), seed_hex.size()},
+          seed_buf.begin()));
       OUTCOME_TRY(public_key, common::unhexWith0x(getParam<2>()));
-      return api_->insertKey(
-          crypto::decodeKeyTypeFromStr(getParam<0>()), seed, public_key);
+      return api_->insertKey(crypto::decodeKeyTypeFromStr(getParam<0>()),
+                             std::move(seed_buf),
+                             public_key);
     }
 
    private:

diff --git a/core/application/app_state_manager.hpp b/core/application/app_state_manager.hpp
@@ -6,7 +6,9 @@
 
 #pragma once
 
-#include "log/logger.hpp"
+#include <functional>
+#include <stdexcept>
+#include <string>
 
 namespace kagome::application {
 

diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp
@@ -1140,7 +1140,8 @@ namespace kagome::application {
     find_argument<std::string>(
         vm, "node-key", [&](const std::string &val) { node_key.emplace(val); });
     if (node_key.has_value()) {
-      auto key_res = crypto::Ed25519Seed::fromHex(node_key.value());
+      auto key_res = crypto::Ed25519Seed::fromHex(
+          crypto::SecureCleanGuard{node_key.value()});
       if (not key_res.has_value()) {
         auto err_msg = fmt::format(
             "Node key '{}' is invalid: {}", node_key.value(), key_res.error());
@@ -1493,7 +1494,8 @@ namespace kagome::application {
       use_pvf_subprocess_ = false;
     }
 
-    if (auto arg = find_argument<uint32_t>(vm, "parachain-check-deadline"); arg.has_value()) {
+    if (auto arg = find_argument<uint32_t>(vm, "parachain-check-deadline");
+        arg.has_value()) {
       pvf_subprocess_deadline_ = std::chrono::milliseconds(*arg);
     }
 

diff --git a/core/assets/CMakeLists.txt b/core/assets/CMakeLists.txt
@@ -26,6 +26,7 @@ embedded_chainspec(embedded_chainspec_polkadot ${PROJECT_SOURCE_DIR}/examples/po
 embedded_chainspec(embedded_chainspec_kusama ${PROJECT_SOURCE_DIR}/examples/kusama/kusama.json)
 embedded_chainspec(embedded_chainspec_rococo ${PROJECT_SOURCE_DIR}/examples/rococo/rococo.json)
 embedded_chainspec(embedded_chainspec_westend ${PROJECT_SOURCE_DIR}/examples/westend/westend.json)
+embedded_chainspec(embedded_chainspec_paseo ${PROJECT_SOURCE_DIR}/examples/paseo/paseo.json)
 
 set(OUTPUT embedded_keys.cpp)
 if(EMBEDDINGS)

diff --git a/core/assets/assets.hpp b/core/assets/assets.hpp
@@ -15,6 +15,7 @@ namespace kagome::assets {
   extern const char *const embedded_chainspec_kusama;
   extern const char *const embedded_chainspec_rococo;
   extern const char *const embedded_chainspec_westend;
+  extern const char *const embedded_chainspec_paseo;
   extern const std::vector<std::pair<const char *, const char *>> embedded_keys;
 
 }  // namespace kagome::assets
diff --git a/core/assets/embedded_chainspec.hpp b/core/assets/embedded_chainspec.hpp
@@ -23,6 +23,8 @@ namespace kagome::assets {
       res = embedded_chainspec_rococo;
     } else if (name == "westend") {
       res = embedded_chainspec_westend;
+    } else if (name == "paseo") {
+      res = embedded_chainspec_paseo;
     }
     if (res == nullptr) {
       return std::nullopt;

diff --git a/core/authority_discovery/publisher/address_publisher.cpp b/core/authority_discovery/publisher/address_publisher.cpp
@@ -56,10 +56,14 @@ namespace kagome::authority_discovery {
     BOOST_ASSERT(scheduler_ != nullptr);
     app_state_manager->atLaunch([this] { return start(); });
     if (libp2p_key.privateKey.type == libp2p::crypto::Key::Type::Ed25519) {
+      std::array<uint8_t, crypto::constants::ed25519::PRIVKEY_SIZE>
+          private_key_bytes{};
+      std::copy_n(libp2p_key.privateKey.data.begin(),
+                  crypto::constants::ed25519::PRIVKEY_SIZE,
+                  private_key_bytes.begin());
       libp2p_key_.emplace(crypto::Ed25519Keypair{
-          .secret_key =
-              crypto::Ed25519PrivateKey::fromSpan(libp2p_key.privateKey.data)
-                  .value(),
+          .secret_key = crypto::Ed25519PrivateKey::from(
+              crypto::SecureCleanGuard(private_key_bytes)),
           .public_key =
               crypto::Ed25519PublicKey::fromSpan(libp2p_key.publicKey.data)
                   .value(),

diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp
@@ -126,7 +126,7 @@ namespace kagome::blockchain {
       std::shared_ptr<const class JustificationStoragePolicy>
           justification_storage_policy,
       std::shared_ptr<storage::trie_pruner::TriePruner> state_pruner,
-      std::shared_ptr<common::MainPoolHandler> main_pool_handler) {
+      common::MainThreadPool &main_thread_pool) {
     BOOST_ASSERT(storage != nullptr);
     BOOST_ASSERT(header_repo != nullptr);
 
@@ -285,7 +285,7 @@ namespace kagome::blockchain {
                           std::move(extrinsic_event_key_repo),
                           std::move(justification_storage_policy),
                           state_pruner,
-                          std::move(main_pool_handler)));
+                          main_thread_pool));
 
     // Add non-finalized block to the block tree
     for (auto &e : collected) {
@@ -421,7 +421,7 @@ namespace kagome::blockchain {
       std::shared_ptr<const JustificationStoragePolicy>
           justification_storage_policy,
       std::shared_ptr<storage::trie_pruner::TriePruner> state_pruner,
-      std::shared_ptr<common::MainPoolHandler> main_pool_handler)
+      common::MainThreadPool &main_thread_pool)
       : block_tree_data_{BlockTreeData{
           .header_repo_ = std::move(header_repo),
           .storage_ = std::move(storage),
@@ -435,7 +435,7 @@ namespace kagome::blockchain {
           .genesis_block_hash_ = {},
           .blocks_pruning_ = {app_config.blocksPruning(), finalized.number},
       }},
-        main_pool_handler_(std::move(main_pool_handler)) {
+        main_pool_handler_{main_thread_pool.handlerStarted()} {
     block_tree_data_.sharedAccess([&](const BlockTreeData &p) {
       BOOST_ASSERT(p.header_repo_ != nullptr);
       BOOST_ASSERT(p.storage_ != nullptr);

diff --git a/core/blockchain/impl/block_tree_impl.hpp b/core/blockchain/impl/block_tree_impl.hpp
@@ -33,14 +33,18 @@
 #include "telemetry/service.hpp"
 #include "utils/safe_object.hpp"
 
+namespace kagome {
+  class PoolHandler;
+}  // namespace kagome
+
 namespace kagome::blockchain {
   struct ReorgAndPrune;
   class TreeNode;
   class CachedTree;
 }  // namespace kagome::blockchain
 
 namespace kagome::common {
-  class MainPoolHandler;
+  class MainThreadPool;
 }
 
 namespace kagome::storage::trie_pruner {
@@ -67,7 +71,7 @@ namespace kagome::blockchain {
         std::shared_ptr<const class JustificationStoragePolicy>
             justification_storage_policy,
         std::shared_ptr<storage::trie_pruner::TriePruner> state_pruner,
-        std::shared_ptr<common::MainPoolHandler> main_pool_handler);
+        common::MainThreadPool &main_thread_pool);
 
     /// Recover block tree state at provided block
     static outcome::result<void> recover(
@@ -199,7 +203,7 @@ namespace kagome::blockchain {
         std::shared_ptr<const class JustificationStoragePolicy>
             justification_storage_policy,
         std::shared_ptr<storage::trie_pruner::TriePruner> state_pruner,
-        std::shared_ptr<common::MainPoolHandler> main_pool_handler);
+        common::MainThreadPool &main_thread_pool);
 
     outcome::result<void> reorgAndPrune(BlockTreeData &p,
                                         const ReorgAndPrune &changes);
@@ -280,7 +284,7 @@ namespace kagome::blockchain {
     metrics::Gauge *metric_best_block_height_;
     metrics::Gauge *metric_finalized_block_height_;
     metrics::Gauge *metric_known_chain_leaves_;
-    std::shared_ptr<common::MainPoolHandler> main_pool_handler_;
+    std::shared_ptr<PoolHandler> main_pool_handler_;
     telemetry::Telemetry telemetry_ = telemetry::createTelemetryService();
   };
 }  // namespace kagome::blockchain
diff --git a/core/common/buffer.hpp b/core/common/buffer.hpp
@@ -25,13 +25,13 @@ namespace kagome::common {
   /**
    * @brief Class represents arbitrary (including empty) byte buffer.
    */
-  template <size_t MaxSize>
-  class SLBuffer : public SLVector<uint8_t, MaxSize> {
+  template <size_t MaxSize, typename Allocator = std::allocator<uint8_t>>
+  class SLBuffer : public SLVector<uint8_t, MaxSize, Allocator> {
    public:
-    using Base = SLVector<uint8_t, MaxSize>;
+    using Base = SLVector<uint8_t, MaxSize, Allocator>;
 
     template <size_t OtherMaxSize>
-    using OtherSLBuffer = SLBuffer<OtherMaxSize>;
+    using OtherSLBuffer = SLBuffer<OtherMaxSize, Allocator>;
 
     SLBuffer() = default;
 
@@ -160,6 +160,11 @@ namespace kagome::common {
       return std::span(*this).subspan(offset, length);
     }
 
+    template <size_t Offset, size_t Length>
+    BufferView view() const {
+      return std::span(*this).template subspan<Offset, Length>();
+    }
+
     /**
      * @brief encode bytearray as hex
      * @return hex-encoded string
@@ -213,7 +218,7 @@ namespace kagome::common {
     return os << BufferView(buffer);
   }
 
-  typedef SLBuffer<std::numeric_limits<size_t>::max()> Buffer;
+  using Buffer = SLBuffer<std::numeric_limits<size_t>::max()>;
 
   inline static const Buffer kEmptyBuffer{};
 

diff --git a/core/common/buffer_view.hpp b/core/common/buffer_view.hpp
@@ -20,7 +20,7 @@ inline auto operator""_bytes(const char *s, std::size_t size) {
 }
 
 namespace kagome::common {
-  template <size_t MaxSize>
+  template <size_t MaxSize, typename Allocator>
   class SLBuffer;
 }