Merge branch 'master' into zombie_builder_rework

# Conflicts: # zombienet/polkadot/functional/0001-parachains-pvf.toml
qdrvm · Jul 4, 2024 · f3a01ac · f3a01ac
2 parents 7c22a6e + 9e2aaf8
commit f3a01ac
Show file tree

Hide file tree

Showing 120 changed files with 1,156 additions and 1,188 deletions.
diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake
@@ -74,15 +74,6 @@ if ("${WASM_COMPILER}" STREQUAL "WAVM")
   )
 endif ()
 
-# Fix for Apple clang (or clang from brew) of versions 15 and higher
-if (APPLE AND (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") AND CMAKE_CXX_COMPILER_VERSION GREATER_EQUAL "15.0.0")
-  hunter_config(
-      binaryen
-      URL https://github.com/qdrvm/binaryen/archive/e6a2fea157bde503f07f28444b350512374cf5bf.zip
-      SHA1 301f8b1775904179cb552c12be237b4aa076981e
-  )
-endif ()
-
 hunter_config(
     libsecp256k1
     VERSION 0.4.1-qdrvm1

diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake
@@ -1,5 +1,5 @@
 HunterGate(
-  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm16.zip
-  SHA1 990ea05207260b3757ce051e354cc163e910a211
+  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm18.zip
+  SHA1 22d842b448f84a39392d7835f4046da34f8dcd70
   LOCAL
 )
diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp
@@ -8,7 +8,6 @@
 
 #include <boost/program_options/value_semantic.hpp>
 #include <charconv>
-#include <filesystem>
 #include <limits>
 #include <regex>
 #include <string>
@@ -31,9 +30,10 @@
 #include "common/hexutil.hpp"
 #include "common/uri.hpp"
 #include "filesystem/common.hpp"
-#include "filesystem/directories.hpp"
 #include "log/formatters/filepath.hpp"
+#include "utils/mkdirs.hpp"
 #include "utils/read_file.hpp"
+#include "utils/write_file.hpp"
 
 namespace {
   namespace fs = kagome::filesystem;
@@ -507,7 +507,7 @@ namespace kagome::application {
       return false;
     }
 
-    if (base_path_.empty() or !fs::createDirectoryRecursive(base_path_)) {
+    if (not mkdirs(base_path_)) {
       SL_ERROR(logger_,
                "Base path {} does not exist, "
                "please specify a valid path with -d option",
@@ -988,13 +988,10 @@ namespace kagome::application {
         }
 
         if (not kagome::filesystem::exists(chain_spec_path_)) {
-          kagome::filesystem::create_directories(
-              chain_spec_path_.parent_path());
+          mkdirs(chain_spec_path_.parent_path()).value();
 
-          std::ofstream ofs;
-          ofs.open(chain_spec_path_.native(), std::ios::ate);
-          ofs << kagome::assets::embedded_chainspec;
-          ofs.close();
+          writeFile(chain_spec_path_, kagome::assets::embedded_chainspec)
+              .value();
 
           auto chain_spec = ChainSpecImpl::loadFrom(chain_spec_path_.native());
           auto path = keystorePath(chain_spec.value()->id());
@@ -1015,12 +1012,10 @@ namespace kagome::application {
           auto ma_res = chain_spec.value()->bootNodes()[0];
           listen_addresses_.emplace_back(ma_res);
 
-          kagome::filesystem::create_directories(path);
+          mkdirs(path).value();
 
           for (auto key_descr : kagome::assets::embedded_keys) {
-            ofs.open((path / key_descr.first).native(), std::ios::ate);
-            ofs << key_descr.second;
-            ofs.close();
+            writeFile(path / key_descr.first, key_descr.second).value();
           }
         }
 
@@ -1476,13 +1471,12 @@ namespace kagome::application {
       }
     }
     {
-      std::error_code ec;
-      kagome::filesystem::create_directories(runtimeCacheDirPath(), ec);
-      if (ec) {
+      auto r = mkdirs(runtimeCacheDirPath());
+      if (not r) {
         SL_ERROR(logger_,
                  "Failed to create runtime cache dir {}: {}",
                  runtimeCacheDirPath(),
-                 ec);
+                 r.error());
         return false;
       }
     }

diff --git a/core/application/modes/precompile_wasm.cpp b/core/application/modes/precompile_wasm.cpp
@@ -8,22 +8,23 @@
 
 #include "blockchain/block_tree.hpp"
 #include "common/bytestr.hpp"
+#include "parachain/pvf/pool.hpp"
 #include "parachain/pvf/session_params.hpp"
-#include "runtime/common/uncompress_code_if_needed.hpp"
-#include "runtime/module_factory.hpp"
-#include "runtime/wabt/instrument.hpp"
+#include "runtime/common/runtime_instances_pool.hpp"
 #include "utils/read_file.hpp"
 
 namespace kagome::application::mode {
   PrecompileWasmMode::PrecompileWasmMode(
       const application::AppConfiguration &app_config,
       std::shared_ptr<blockchain::BlockTree> block_tree,
       std::shared_ptr<runtime::ParachainHost> parachain_api,
-      std::shared_ptr<runtime::ModuleFactory> module_factory)
+      std::shared_ptr<crypto::Hasher> hasher,
+      std::shared_ptr<parachain::PvfPool> module_factory)
       : log_{log::createLogger("PrecompileWasm")},
         config_{*app_config.precompileWasm()},
         block_tree_{std::move(block_tree)},
         parachain_api_{std::move(parachain_api)},
+        hasher_{std::move(hasher)},
         module_factory_{std::move(module_factory)} {}
 
   int PrecompileWasmMode::run() const {
@@ -55,13 +56,11 @@ namespace kagome::application::mode {
       if (text.starts_with("0x")) {
         BOOST_OUTCOME_TRY(bytes, common::unhexWith0x(text));
       }
-      Buffer code;
-      OUTCOME_TRY(runtime::uncompressCodeIfNeeded(bytes, code));
+      // https://github.com/paritytech/polkadot-sdk/blob/b4ae5b01da280f754ccc00b94314a30b658182a1/polkadot/parachain/src/primitives.rs#L74-L81
+      auto code_hash = hasher_->blake2b_256(bytes);
       OUTCOME_TRY(config,
                   parachain::sessionParams(*parachain_api_, block.hash));
-      BOOST_OUTCOME_TRY(
-          code, runtime::prepareBlobForCompilation(code, config.memory_limits));
-      OUTCOME_TRY(module_factory_->make(code));
+      OUTCOME_TRY(module_factory_->precompile(code_hash, bytes, config));
     }
     return outcome::success();
   }

diff --git a/core/application/modes/precompile_wasm.hpp b/core/application/modes/precompile_wasm.hpp
@@ -14,8 +14,15 @@ namespace kagome::blockchain {
   class BlockTree;
 }  // namespace kagome::blockchain
 
+namespace kagome::crypto {
+  class Hasher;
+}  // namespace kagome::crypto
+
+namespace kagome::parachain {
+  class PvfPool;
+}  // namespace kagome::parachain
+
 namespace kagome::runtime {
-  class ModuleFactory;
   class ParachainHost;
 }  // namespace kagome::runtime
 
@@ -28,7 +35,8 @@ namespace kagome::application::mode {
     PrecompileWasmMode(const application::AppConfiguration &app_config,
                        std::shared_ptr<blockchain::BlockTree> block_tree,
                        std::shared_ptr<runtime::ParachainHost> parachain_api,
-                       std::shared_ptr<runtime::ModuleFactory> module_factory);
+                       std::shared_ptr<crypto::Hasher> hasher,
+                       std::shared_ptr<parachain::PvfPool> module_factory);
 
     int run() const override;
 
@@ -39,6 +47,7 @@ namespace kagome::application::mode {
     application::PrecompileWasmConfig config_;
     std::shared_ptr<blockchain::BlockTree> block_tree_;
     std::shared_ptr<runtime::ParachainHost> parachain_api_;
-    std::shared_ptr<runtime::ModuleFactory> module_factory_;
+    std::shared_ptr<crypto::Hasher> hasher_;
+    std::shared_ptr<parachain::PvfPool> module_factory_;
   };
 }  // namespace kagome::application::mode
diff --git a/core/consensus/timeline/impl/block_executor_impl.cpp b/core/consensus/timeline/impl/block_executor_impl.cpp
@@ -20,6 +20,7 @@
 #include "storage/changes_trie/impl/storage_changes_tracker_impl.hpp"
 #include "transaction_pool/transaction_pool.hpp"
 #include "transaction_pool/transaction_pool_error.hpp"
+#include "utils/pool_handler_ready_make.hpp"
 
 namespace kagome::consensus {
 
@@ -43,7 +44,8 @@ namespace kagome::consensus {
       std::unique_ptr<BlockAppenderBase> appender)
       : block_tree_{std::move(block_tree)},
         main_pool_handler_{main_thread_pool.handler(app_state_manager)},
-        worker_pool_handler_{worker_thread_pool.handler(app_state_manager)},
+        worker_pool_handler_{
+            poolHandlerReadyMake(app_state_manager, worker_thread_pool)},
         core_{std::move(core)},
         tx_pool_{std::move(tx_pool)},
         hasher_{std::move(hasher)},

diff --git a/core/consensus/timeline/impl/block_executor_impl.hpp b/core/consensus/timeline/impl/block_executor_impl.hpp
@@ -18,6 +18,7 @@
 
 namespace kagome {
   class PoolHandler;
+  class PoolHandlerReady;
 }  // namespace kagome
 
 namespace kagome::application {
@@ -86,7 +87,7 @@ namespace kagome::consensus {
    private:
     std::shared_ptr<blockchain::BlockTree> block_tree_;
     std::shared_ptr<PoolHandler> main_pool_handler_;
-    std::shared_ptr<PoolHandler> worker_pool_handler_;
+    std::shared_ptr<PoolHandlerReady> worker_pool_handler_;
     std::shared_ptr<runtime::Core> core_;
     std::shared_ptr<transaction_pool::TransactionPool> tx_pool_;
     std::shared_ptr<crypto::Hasher> hasher_;

diff --git a/core/consensus/timeline/impl/timeline_impl.cpp b/core/consensus/timeline/impl/timeline_impl.cpp
@@ -10,6 +10,7 @@
 #include "application/app_state_manager.hpp"
 #include "blockchain/block_tree.hpp"
 #include "clock/impl/clock_impl.hpp"
+#include "common/main_thread_pool.hpp"
 #include "consensus/consensus_selector.hpp"
 #include "consensus/grandpa/justification_observer.hpp"
 #include "consensus/timeline/impl/slot_leadership_error.hpp"
@@ -37,6 +38,7 @@ namespace kagome::consensus {
       const application::AppConfiguration &app_config,
       std::shared_ptr<application::AppStateManager> app_state_manager,
       const clock::SystemClock &clock,
+      common::MainThreadPool &main_thread_pool,
       std::shared_ptr<SlotsUtil> slots_util,
       std::shared_ptr<blockchain::BlockTree> block_tree,
       std::shared_ptr<ConsensusSelector> consensus_selector,
@@ -57,6 +59,7 @@ namespace kagome::consensus {
       : log_(log::createLogger("Timeline", "timeline")),
         app_state_manager_(std::move(app_state_manager)),
         clock_(clock),
+        main_pool_handler_{main_thread_pool.handlerStarted()},
         slots_util_(std::move(slots_util)),
         block_tree_(std::move(block_tree)),
         consensus_selector_(std::move(consensus_selector)),
@@ -272,7 +275,13 @@ namespace kagome::consensus {
 
       if (validator_status == ValidatorStatus::SingleValidator) {
         SL_INFO(log_, "Starting single validating node.");
-        onSynchronized();
+        main_pool_handler_->execute([weak_self{weak_from_this()}] {
+          auto self = weak_self.lock();
+          if (not self) {
+            return;
+          }
+          self->onSynchronized();
+        });
         return true;
       }
     }

diff --git a/core/consensus/timeline/impl/timeline_impl.hpp b/core/consensus/timeline/impl/timeline_impl.hpp
@@ -18,6 +18,10 @@
 #include "primitives/event_types.hpp"
 #include "telemetry/service.hpp"
 
+namespace kagome {
+  class PoolHandler;
+}  // namespace kagome
+
 namespace kagome::application {
   class AppStateManager;
   class AppConfiguration;
@@ -27,6 +31,10 @@ namespace kagome::blockchain {
   class BlockTree;
 }
 
+namespace kagome::common {
+  class MainThreadPool;
+}  // namespace kagome::common
+
 namespace kagome::consensus {
   class SlotsUtil;
   class ConsensusSelector;
@@ -71,6 +79,7 @@ namespace kagome::consensus {
         const application::AppConfiguration &app_config,
         std::shared_ptr<application::AppStateManager> app_state_manager,
         const clock::SystemClock &clock,
+        common::MainThreadPool &main_thread_pool,
         std::shared_ptr<SlotsUtil> slots_util,
         std::shared_ptr<blockchain::BlockTree> block_tree,
         std::shared_ptr<ConsensusSelector> consensus_selector,
@@ -138,6 +147,7 @@ namespace kagome::consensus {
 
     std::shared_ptr<application::AppStateManager> app_state_manager_;
     const clock::SystemClock &clock_;
+    std::shared_ptr<PoolHandler> main_pool_handler_;
     std::shared_ptr<SlotsUtil> slots_util_;
     std::shared_ptr<blockchain::BlockTree> block_tree_;
     std::shared_ptr<ConsensusSelector> consensus_selector_;

diff --git a/core/crypto/key_store/key_file_storage.cpp b/core/crypto/key_store/key_file_storage.cpp
@@ -6,13 +6,13 @@
 
 #include "crypto/key_store/key_file_storage.hpp"
 
-#include <fstream>
-
 #include "common/hexutil.hpp"
 #include "crypto/key_store/key_type.hpp"
 #include "filesystem/common.hpp"
 #include "utils/json_unquote.hpp"
+#include "utils/mkdirs.hpp"
 #include "utils/read_file.hpp"
+#include "utils/write_file.hpp"
 
 OUTCOME_CPP_DEFINE_CATEGORY(kagome::crypto, KeyFileStorage::Error, e) {
   using E = kagome::crypto::KeyFileStorage::Error;
@@ -29,10 +29,6 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::crypto, KeyFileStorage::Error, e) {
       return "specified key file is invalid";
     case E::INCONSISTENT_KEYFILE:
       return "key file is inconsistent, public key != derived public key";
-    case E::KEYS_PATH_IS_NOT_DIRECTORY:
-      return "specified key storage directory path is not a directory";
-    case E::FAILED_CREATE_KEYS_DIRECTORY:
-      return "failed to create key storage directory";
   }
   return "unknown KeyFileStorage error";
 }
@@ -80,44 +76,13 @@ namespace kagome::crypto {
   }
 
   outcome::result<void> KeyFileStorage::initialize() {
-    std::error_code ec{};
-    bool does_exist = filesystem::exists(keystore_path_, ec);
-    if (ec and ec != std::errc::no_such_file_or_directory) {
-      logger_->error("Error initializing key storage: {}", ec);
-      return outcome::failure(ec);
-    }
-    if (does_exist) {
-      // check whether specified path is a directory
-      if (not filesystem::is_directory(keystore_path_, ec)) {
-        return Error::KEYS_PATH_IS_NOT_DIRECTORY;
-      }
-      if (ec) {
-        logger_->error("Error scanning key storage: {}", ec);
-        return outcome::failure(ec);
-      }
-    } else {
-      // try create directory
-      if (not filesystem::create_directories(keystore_path_, ec)) {
-        return Error::FAILED_CREATE_KEYS_DIRECTORY;
-      }
-      if (ec) {
-        logger_->error("Error creating keystore dir: {}", ec);
-        return outcome::failure(ec);
-      }
-    }
-
+    OUTCOME_TRY(mkdirs(keystore_path_));
     return outcome::success();
   }
 
   outcome::result<void> KeyFileStorage::saveKeyHexAtPath(
       common::BufferView private_key, const KeyFileStorage::Path &path) const {
-    std::ofstream file;
-    file.open(path.native(), std::ios::out | std::ios::trunc);
-    if (!file.is_open()) {
-      return Error::FAILED_OPEN_FILE;
-    }
-    auto hex = common::hex_lower_0x(private_key);
-    file << hex;
+    OUTCOME_TRY(writeFile(path, common::hex_lower_0x(private_key)));
     SL_TRACE(logger_, "Saving key to {}", path.native());
     return outcome::success();
   }

diff --git a/core/crypto/key_store/key_file_storage.hpp b/core/crypto/key_store/key_file_storage.hpp
@@ -30,8 +30,6 @@ namespace kagome::crypto {
       FILE_DOESNT_EXIST,
       INVALID_FILE_FORMAT,
       INCONSISTENT_KEYFILE,
-      KEYS_PATH_IS_NOT_DIRECTORY,
-      FAILED_CREATE_KEYS_DIRECTORY
     };
 
     using Buffer = common::Buffer;