diff --git a/core/application/impl/kagome_application_impl.cpp b/core/application/impl/kagome_application_impl.cpp index eaab11dd0b..0fde7f639e 100644 --- a/core/application/impl/kagome_application_impl.cpp +++ b/core/application/impl/kagome_application_impl.cpp @@ -126,7 +126,7 @@ namespace kagome::application { if (not app_config_->disableSecureMode() and app_config_->usePvfSubprocess() and app_config_->roles().isAuthority()) { auto res = parachain::runSecureModeCheckProcess( - *injector_.injectIoContext(), app_config_->runtimeCacheDirPath()); + app_config_->runtimeCacheDirPath()); if (!res) { SL_ERROR(logger_, "Secure mode check failed: {}", res.error()); exit(EXIT_FAILURE); diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 3a38482936..9fe8669a72 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -194,6 +194,7 @@ #include "runtime/runtime_api/impl/transaction_payment_api.hpp" #include "runtime/wabt/instrument.hpp" #include "runtime/wasm_compiler_definitions.hpp" // this header-file is generated +#include "utils/sptr.hpp" #if KAGOME_WASM_COMPILER_WASM_EDGE == 1 @@ -790,6 +791,32 @@ namespace { di::bind.template to(), di::bind.template to(), di::bind.template to(), + bind_by_lambda([config](const auto &) { + auto support = parachain::SecureModeSupport::none(); + auto log = log::createLogger("Application", "application"); +#ifdef __linux__ + if (not config->disableSecureMode() and config->usePvfSubprocess() + and config->roles().isAuthority()) { + auto res = parachain::runSecureModeCheckProcess(config->runtimeCacheDirPath()); + if (!res) { + SL_ERROR(log, "Secure mode check failed: {}", res.error()); + exit(EXIT_FAILURE); + } + support = res.value(); + if (not support.isTotallySupported()) { + SL_ERROR(log, + "Secure mode is not supported completely. You can disable it " + "using --insecure-validator-i-know-what-i-do."); + exit(EXIT_FAILURE); + } + } +#else + SL_WARN(log, + "Secure validator mode is not implemented for the current " + "platform. Proceed at your own risk."); +#endif + return toSptr(support); + }), di::bind.template to(), di::bind.template to(), di::bind.template to(), @@ -936,7 +963,7 @@ namespace kagome::injector { KagomeNodeInjector::KagomeNodeInjector( sptr app_config) : pimpl_{std::make_unique( - makeKagomeNodeInjector(std::move(app_config)))} {} + makeKagomeNodeInjector(std::move(app_config)))} {} sptr KagomeNodeInjector::injectAppConfig() { return pimpl_->injector_ diff --git a/core/parachain/pvf/clone.hpp b/core/parachain/pvf/clone.hpp new file mode 100644 index 0000000000..c6352acf85 --- /dev/null +++ b/core/parachain/pvf/clone.hpp @@ -0,0 +1,132 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +#ifdef __linux__ +#include +#include +#endif + +#include "parachain/pvf/pvf_worker_types.hpp" + +namespace kagome::parachain::clone { + constexpr size_t kCloneStackSize = 2 << 20; + + enum class CloneError : uint8_t { + kCallbackFailed, + }; + Q_ENUM_ERROR_CODE(CloneError) { + using E = decltype(e); + switch (e) { + case E::kCallbackFailed: + return "Callback failed"; + } + abort(); + } + +#ifdef __linux__ + // https://github.com/paritytech/polkadot-sdk/blob/f4a196ab1473856c9c5992239fcc2f14c2c42914/polkadot/node/core/pvf/common/src/worker/security/clone.rs#L35-L54 + /// Try to run clone(2) on the current worker. + /// + /// SAFETY: new process should be either spawned within a single threaded + /// process, or use only async-signal-safe functions. + template + inline outcome::result clone(bool have_unshare_newuser, const Cb &cb) { + Buffer stack(kCloneStackSize); + // https://github.com/paritytech/polkadot-sdk/blob/f4a196ab1473856c9c5992239fcc2f14c2c42914/polkadot/node/core/pvf/common/src/worker/security/clone.rs#L75-L93 + int flags = CLONE_NEWCGROUP | CLONE_NEWIPC | CLONE_NEWNET | CLONE_NEWNS + | CLONE_NEWPID | CLONE_NEWUTS | SIGCHLD; + if (not have_unshare_newuser) { + flags |= CLONE_NEWUSER; + } + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + pid_t pid = ::clone( + [](void *arg) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto &cb = *reinterpret_cast(arg); + return cb() ? EXIT_SUCCESS : EXIT_FAILURE; + }, + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + stack.data() + stack.size(), + flags, + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + const_cast(static_cast(&cb))); + if (pid == -1) { + return std::errc{errno}; + } + return pid; + } +#endif + + inline outcome::result wait(pid_t pid) { + int status = 0; + if (waitpid(pid, &status, 0) == -1) { + return std::errc{errno}; + } + if (not WIFEXITED(status) or WEXITSTATUS(status) != EXIT_SUCCESS) { + return CloneError::kCallbackFailed; + } + return outcome::success(); + } + + // https://github.com/paritytech/polkadot-sdk/blob/f4a196ab1473856c9c5992239fcc2f14c2c42914/polkadot/node/core/pvf/execute-worker/src/lib.rs#L245-L293 + /// Call callback either directly, or inside `clone`, or inside `fork`. + inline outcome::result cloneOrFork(const log::Logger &log, + const PvfWorkerInputConfig &config, + const auto &cb) { + auto cb_log = [&] { + auto r = cb(); + if (not r) { + SL_WARN(log, "cloneOrFork cb returned error: {}", r.error()); + return false; + } + return true; + }; + if (config.force_disable_secure_mode) { + if (not cb_log()) { + return CloneError::kCallbackFailed; + } + return outcome::success(); + } + std::optional pid; +#ifdef __linux__ + if (config.secure_mode_support.can_do_secure_clone) { + BOOST_OUTCOME_TRY(pid, clone(config.secure_mode_support.chroot, cb_log)); + } +#endif + if (not pid) { + pid = fork(); + if (pid == -1) { + return std::errc{errno}; + } + if (pid == 0) { + _Exit(cb_log() ? EXIT_SUCCESS : EXIT_FAILURE); + } + } + return wait(*pid); + } + + // https://github.com/paritytech/polkadot-sdk/blob/f4a196ab1473856c9c5992239fcc2f14c2c42914/polkadot/node/core/pvf/common/src/worker/security/clone.rs#L56-L63 + /// Runs a check for clone(2) with all sandboxing flags and returns an error + /// indicating whether it can be fully enabled on the current Linux + /// environment. + /// + /// SAFETY: new process should be either spawned within a single threaded + /// process, or use only async-signal-safe functions. + inline outcome::result check() { +#ifdef __linux__ + OUTCOME_TRY(pid, clone(false, [] { return true; })); + return wait(pid); +#else + return std::errc::not_supported; +#endif + } +} // namespace kagome::parachain::clone diff --git a/core/parachain/pvf/kagome_pvf_worker.cpp b/core/parachain/pvf/kagome_pvf_worker.cpp index c431086b2b..49e04d11cd 100644 --- a/core/parachain/pvf/kagome_pvf_worker.cpp +++ b/core/parachain/pvf/kagome_pvf_worker.cpp @@ -34,6 +34,7 @@ #include "common/bytestr.hpp" #include "log/configurator.hpp" #include "log/logger.hpp" +#include "parachain/pvf/clone.hpp" #include "parachain/pvf/kagome_pvf_worker.hpp" #include "parachain/pvf/kagome_pvf_worker_injector.hpp" #include "parachain/pvf/pvf_worker_types.hpp" @@ -366,23 +367,27 @@ namespace kagome::parachain { SL_ERROR(logger, "PvfWorkerInputCodeParams expected"); return std::errc::invalid_argument; } - OUTCOME_TRY(instance, module->instantiate()); - - OUTCOME_TRY(ctx, runtime::RuntimeContextFactory::stateless(instance)); - OUTCOME_TRY( - result, - instance->callExportFunction(ctx, "validate_block", input_args)); - OUTCOME_TRY(instance->resetEnvironment()); - OUTCOME_TRY(len, scale::encode(result.size())); - - boost::asio::write(socket, boost::asio::buffer(len), ec); - if (ec) { - return ec; - } - boost::asio::write(socket, boost::asio::buffer(result), ec); - if (ec) { - return ec; - } + auto forked = [&]() -> outcome::result { + OUTCOME_TRY(instance, module->instantiate()); + + OUTCOME_TRY(ctx, runtime::RuntimeContextFactory::stateless(instance)); + OUTCOME_TRY( + result, + instance->callExportFunction(ctx, "validate_block", input_args)); + OUTCOME_TRY(instance->resetEnvironment()); + OUTCOME_TRY(len, scale::encode(result.size())); + + boost::asio::write(socket, boost::asio::buffer(len), ec); + if (ec) { + return ec; + } + boost::asio::write(socket, boost::asio::buffer(result), ec); + if (ec) { + return ec; + } + return outcome::success(); + }; + OUTCOME_TRY(clone::cloneOrFork(logger, input_config, forked)); } } diff --git a/core/parachain/pvf/pvf_worker_types.hpp b/core/parachain/pvf/pvf_worker_types.hpp index dad1bd6189..e4b9b37250 100644 --- a/core/parachain/pvf/pvf_worker_types.hpp +++ b/core/parachain/pvf/pvf_worker_types.hpp @@ -9,6 +9,7 @@ #include #include "common/buffer.hpp" +#include "parachain/pvf/secure_mode_precheck.hpp" #include "runtime/runtime_context.hpp" #include "scale/scale.hpp" #include "scale/std_variant.hpp" @@ -31,12 +32,13 @@ namespace kagome::parachain { const application::AppConfiguration &app_config); struct PvfWorkerInputConfig { - SCALE_TIE(4); + SCALE_TIE(5); RuntimeEngine engine; std::string cache_dir; std::vector log_params; bool force_disable_secure_mode; + SecureModeSupport secure_mode_support; }; struct PvfWorkerInputCodeParams { diff --git a/core/parachain/pvf/secure_mode.hpp b/core/parachain/pvf/secure_mode.hpp index 617370360f..3100b1d2ee 100644 --- a/core/parachain/pvf/secure_mode.hpp +++ b/core/parachain/pvf/secure_mode.hpp @@ -22,6 +22,11 @@ namespace kagome::parachain { std::string message_; }; + + inline auto make_exception_ptr(SecureModeError e) { + return std::make_exception_ptr(std::move(e)); + } + template using SecureModeOutcome = CustomOutcome; diff --git a/core/parachain/pvf/secure_mode_precheck.cpp b/core/parachain/pvf/secure_mode_precheck.cpp index a673bc3cfc..d60a70f9af 100644 --- a/core/parachain/pvf/secure_mode_precheck.cpp +++ b/core/parachain/pvf/secure_mode_precheck.cpp @@ -23,6 +23,7 @@ #include "common/buffer_view.hpp" #include "log/configurator.hpp" #include "log/logger.hpp" +#include "parachain/pvf/clone.hpp" #include "parachain/pvf/secure_mode.hpp" #include "utils/get_exe_path.hpp" @@ -36,8 +37,17 @@ namespace kagome::parachain { std::filesystem::path cache_dir = original_cache_dir; SecureModeSupport support = SecureModeSupport::none(); auto logger = log::createLogger("CheckSecureMode", "parachain"); + if (auto res = clone::check()) { + support.can_do_secure_clone = true; + } else { + SL_WARN(logger, + "Secure mode incomplete, cannot enable clone for PVF " + "worker: {}", + res.error()); + } if (auto res = changeRoot(cache_dir)) { support.chroot = true; + cache_dir = "/"; } else { SL_WARN( logger, @@ -46,7 +56,6 @@ namespace kagome::parachain { cache_dir.c_str(), res.error()); } - cache_dir = "/"; if (auto res = enableLandlock(cache_dir)) { support.landlock = true; @@ -70,8 +79,8 @@ namespace kagome::parachain { } SecureModeOutcome runSecureModeCheckProcess( - boost::asio::io_context &io_context, const std::filesystem::path &cache_dir) { + boost::asio::io_context io_context; namespace process_v2 = boost::process::v2; boost::asio::readable_pipe pipe{io_context}; // input passed as CLI arguments to enable users to manually run the check diff --git a/core/parachain/pvf/secure_mode_precheck.hpp b/core/parachain/pvf/secure_mode_precheck.hpp index 9d4f5d0dc3..6802d462e8 100644 --- a/core/parachain/pvf/secure_mode_precheck.hpp +++ b/core/parachain/pvf/secure_mode_precheck.hpp @@ -6,7 +6,6 @@ #pragma once -#include #include #include "parachain/pvf/secure_mode.hpp" @@ -20,7 +19,7 @@ namespace kagome::parachain { * platform */ struct SecureModeSupport { - SCALE_TIE(3); + SCALE_TIE(4); // The filesystem root of the PVF process can be set to the worker directory bool chroot; @@ -32,8 +31,16 @@ namespace kagome::parachain { // process bool seccomp; + // Whether we are able to call `clone` with all sandboxing flags. + bool can_do_secure_clone; + static SecureModeSupport none() { - return {false, false, false}; + return { + .chroot = false, + .landlock = false, + .seccomp = false, + .can_do_secure_clone = false, + }; } bool isTotallySupported() const { @@ -51,7 +58,6 @@ namespace kagome::parachain { * Spawns a child process that executes checkSecureMode */ SecureModeOutcome runSecureModeCheckProcess( - boost::asio::io_context &io_context, const std::filesystem::path &cache_dir); /** diff --git a/core/parachain/pvf/workers.cpp b/core/parachain/pvf/workers.cpp index 4aaffafb95..0d0fc7fa8b 100644 --- a/core/parachain/pvf/workers.cpp +++ b/core/parachain/pvf/workers.cpp @@ -111,6 +111,7 @@ namespace kagome::parachain { PvfWorkers::PvfWorkers(const application::AppConfiguration &app_config, common::MainThreadPool &main_thread_pool, + SecureModeSupport secure_mode_support, std::shared_ptr scheduler) : io_context_{main_thread_pool.io_context()}, main_pool_handler_{main_thread_pool.handlerStarted()}, @@ -122,6 +123,7 @@ namespace kagome::parachain { .cache_dir = app_config.runtimeCacheDirPath(), .log_params = app_config.log(), .force_disable_secure_mode = app_config.disableSecureMode(), + .secure_mode_support = secure_mode_support, } { metrics_registry_->registerGaugeFamily(kMetricQueueSize, "pvf queue size"); std::unordered_map kind_name{ diff --git a/core/parachain/pvf/workers.hpp b/core/parachain/pvf/workers.hpp index b9dcbde5ad..d7d4ecd837 100644 --- a/core/parachain/pvf/workers.hpp +++ b/core/parachain/pvf/workers.hpp @@ -43,6 +43,7 @@ namespace kagome::parachain { public: PvfWorkers(const application::AppConfiguration &app_config, common::MainThreadPool &main_thread_pool, + SecureModeSupport secure_mode_support, std::shared_ptr scheduler); using Cb = std::function)>; diff --git a/core/utils/sptr.hpp b/core/utils/sptr.hpp new file mode 100644 index 0000000000..d793e1c81b --- /dev/null +++ b/core/utils/sptr.hpp @@ -0,0 +1,16 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace kagome { + template + auto toSptr(T &&t) { + return std::make_shared>(std::forward(t)); + } +} // namespace kagome