Skip to content

Commit

Permalink
pvf secure clone (#2307)
Browse files Browse the repository at this point in the history
Signed-off-by: turuslan <[email protected]>
  • Loading branch information
turuslan authored Dec 26, 2024
1 parent bc110f7 commit 1f04f5d
Show file tree
Hide file tree
Showing 11 changed files with 231 additions and 26 deletions.
2 changes: 1 addition & 1 deletion core/application/impl/kagome_application_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ namespace kagome::application {
if (not app_config_->disableSecureMode() and app_config_->usePvfSubprocess()
and app_config_->roles().isAuthority()) {
auto res = parachain::runSecureModeCheckProcess(
*injector_.injectIoContext(), app_config_->runtimeCacheDirPath());
app_config_->runtimeCacheDirPath());
if (!res) {
SL_ERROR(logger_, "Secure mode check failed: {}", res.error());
exit(EXIT_FAILURE);
Expand Down
29 changes: 28 additions & 1 deletion core/injector/application_injector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@
#include "runtime/runtime_api/impl/transaction_payment_api.hpp"
#include "runtime/wabt/instrument.hpp"
#include "runtime/wasm_compiler_definitions.hpp" // this header-file is generated
#include "utils/sptr.hpp"

#if KAGOME_WASM_COMPILER_WASM_EDGE == 1

Expand Down Expand Up @@ -790,6 +791,32 @@ namespace {
di::bind<parachain::BackedCandidatesSource>.template to<parachain::ParachainProcessorImpl>(),
di::bind<network::CanDisconnect>.template to<parachain::statement_distribution::StatementDistribution>(),
di::bind<parachain::Pvf>.template to<parachain::PvfImpl>(),
bind_by_lambda<parachain::SecureModeSupport>([config](const auto &) {
auto support = parachain::SecureModeSupport::none();
auto log = log::createLogger("Application", "application");
#ifdef __linux__
if (not config->disableSecureMode() and config->usePvfSubprocess()
and config->roles().isAuthority()) {
auto res = parachain::runSecureModeCheckProcess(config->runtimeCacheDirPath());
if (!res) {
SL_ERROR(log, "Secure mode check failed: {}", res.error());
exit(EXIT_FAILURE);
}
support = res.value();
if (not support.isTotallySupported()) {
SL_ERROR(log,
"Secure mode is not supported completely. You can disable it "
"using --insecure-validator-i-know-what-i-do.");
exit(EXIT_FAILURE);
}
}
#else
SL_WARN(log,
"Secure validator mode is not implemented for the current "
"platform. Proceed at your own risk.");
#endif
return toSptr(support);
}),
di::bind<network::CollationObserver>.template to<parachain::ParachainObserverImpl>(),
di::bind<network::ValidationObserver>.template to<parachain::ParachainObserverImpl>(),
di::bind<network::ReqCollationObserver>.template to<parachain::ParachainObserverImpl>(),
Expand Down Expand Up @@ -936,7 +963,7 @@ namespace kagome::injector {
KagomeNodeInjector::KagomeNodeInjector(
sptr<application::AppConfiguration> app_config)
: pimpl_{std::make_unique<KagomeNodeInjectorImpl>(
makeKagomeNodeInjector(std::move(app_config)))} {}
makeKagomeNodeInjector(std::move(app_config)))} {}

sptr<application::AppConfiguration> KagomeNodeInjector::injectAppConfig() {
return pimpl_->injector_
Expand Down
132 changes: 132 additions & 0 deletions core/parachain/pvf/clone.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/**
* Copyright Quadrivium LLC
* All Rights Reserved
* SPDX-License-Identifier: Apache-2.0
*/

#pragma once

#include <sys/wait.h>
#include <unistd.h>
#include <cerrno>

#ifdef __linux__
#include <sched.h>
#include <csignal>
#endif

#include "parachain/pvf/pvf_worker_types.hpp"

namespace kagome::parachain::clone {
constexpr size_t kCloneStackSize = 2 << 20;

enum class CloneError : uint8_t {
kCallbackFailed,
};
Q_ENUM_ERROR_CODE(CloneError) {
using E = decltype(e);
switch (e) {
case E::kCallbackFailed:
return "Callback failed";
}
abort();
}

#ifdef __linux__
// https://github.com/paritytech/polkadot-sdk/blob/f4a196ab1473856c9c5992239fcc2f14c2c42914/polkadot/node/core/pvf/common/src/worker/security/clone.rs#L35-L54
/// Try to run clone(2) on the current worker.
///
/// SAFETY: new process should be either spawned within a single threaded
/// process, or use only async-signal-safe functions.
template <typename Cb>
inline outcome::result<pid_t> clone(bool have_unshare_newuser, const Cb &cb) {
Buffer stack(kCloneStackSize);
// https://github.com/paritytech/polkadot-sdk/blob/f4a196ab1473856c9c5992239fcc2f14c2c42914/polkadot/node/core/pvf/common/src/worker/security/clone.rs#L75-L93
int flags = CLONE_NEWCGROUP | CLONE_NEWIPC | CLONE_NEWNET | CLONE_NEWNS
| CLONE_NEWPID | CLONE_NEWUTS | SIGCHLD;
if (not have_unshare_newuser) {
flags |= CLONE_NEWUSER;
}
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
pid_t pid = ::clone(
[](void *arg) {
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
auto &cb = *reinterpret_cast<const Cb *>(arg);
return cb() ? EXIT_SUCCESS : EXIT_FAILURE;
},
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
stack.data() + stack.size(),
flags,
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
const_cast<void *>(static_cast<const void *>(&cb)));
if (pid == -1) {
return std::errc{errno};
}
return pid;
}
#endif

inline outcome::result<void> wait(pid_t pid) {
int status = 0;
if (waitpid(pid, &status, 0) == -1) {
return std::errc{errno};
}
if (not WIFEXITED(status) or WEXITSTATUS(status) != EXIT_SUCCESS) {
return CloneError::kCallbackFailed;
}
return outcome::success();
}

// https://github.com/paritytech/polkadot-sdk/blob/f4a196ab1473856c9c5992239fcc2f14c2c42914/polkadot/node/core/pvf/execute-worker/src/lib.rs#L245-L293
/// Call callback either directly, or inside `clone`, or inside `fork`.
inline outcome::result<void> cloneOrFork(const log::Logger &log,
const PvfWorkerInputConfig &config,
const auto &cb) {
auto cb_log = [&] {
auto r = cb();
if (not r) {
SL_WARN(log, "cloneOrFork cb returned error: {}", r.error());
return false;
}
return true;
};
if (config.force_disable_secure_mode) {
if (not cb_log()) {
return CloneError::kCallbackFailed;
}
return outcome::success();
}
std::optional<pid_t> pid;
#ifdef __linux__
if (config.secure_mode_support.can_do_secure_clone) {
BOOST_OUTCOME_TRY(pid, clone(config.secure_mode_support.chroot, cb_log));
}
#endif
if (not pid) {
pid = fork();
if (pid == -1) {
return std::errc{errno};
}
if (pid == 0) {
_Exit(cb_log() ? EXIT_SUCCESS : EXIT_FAILURE);
}
}
return wait(*pid);
}

// https://github.com/paritytech/polkadot-sdk/blob/f4a196ab1473856c9c5992239fcc2f14c2c42914/polkadot/node/core/pvf/common/src/worker/security/clone.rs#L56-L63
/// Runs a check for clone(2) with all sandboxing flags and returns an error
/// indicating whether it can be fully enabled on the current Linux
/// environment.
///
/// SAFETY: new process should be either spawned within a single threaded
/// process, or use only async-signal-safe functions.
inline outcome::result<void> check() {
#ifdef __linux__
OUTCOME_TRY(pid, clone(false, [] { return true; }));
return wait(pid);
#else
return std::errc::not_supported;
#endif
}
} // namespace kagome::parachain::clone
39 changes: 22 additions & 17 deletions core/parachain/pvf/kagome_pvf_worker.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include "common/bytestr.hpp"
#include "log/configurator.hpp"
#include "log/logger.hpp"
#include "parachain/pvf/clone.hpp"
#include "parachain/pvf/kagome_pvf_worker.hpp"
#include "parachain/pvf/kagome_pvf_worker_injector.hpp"
#include "parachain/pvf/pvf_worker_types.hpp"
Expand Down Expand Up @@ -366,23 +367,27 @@ namespace kagome::parachain {
SL_ERROR(logger, "PvfWorkerInputCodeParams expected");
return std::errc::invalid_argument;
}
OUTCOME_TRY(instance, module->instantiate());

OUTCOME_TRY(ctx, runtime::RuntimeContextFactory::stateless(instance));
OUTCOME_TRY(
result,
instance->callExportFunction(ctx, "validate_block", input_args));
OUTCOME_TRY(instance->resetEnvironment());
OUTCOME_TRY(len, scale::encode<uint32_t>(result.size()));

boost::asio::write(socket, boost::asio::buffer(len), ec);
if (ec) {
return ec;
}
boost::asio::write(socket, boost::asio::buffer(result), ec);
if (ec) {
return ec;
}
auto forked = [&]() -> outcome::result<void> {
OUTCOME_TRY(instance, module->instantiate());

OUTCOME_TRY(ctx, runtime::RuntimeContextFactory::stateless(instance));
OUTCOME_TRY(
result,
instance->callExportFunction(ctx, "validate_block", input_args));
OUTCOME_TRY(instance->resetEnvironment());
OUTCOME_TRY(len, scale::encode<uint32_t>(result.size()));

boost::asio::write(socket, boost::asio::buffer(len), ec);
if (ec) {
return ec;
}
boost::asio::write(socket, boost::asio::buffer(result), ec);
if (ec) {
return ec;
}
return outcome::success();
};
OUTCOME_TRY(clone::cloneOrFork(logger, input_config, forked));
}
}

Expand Down
4 changes: 3 additions & 1 deletion core/parachain/pvf/pvf_worker_types.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <filesystem>

#include "common/buffer.hpp"
#include "parachain/pvf/secure_mode_precheck.hpp"
#include "runtime/runtime_context.hpp"
#include "scale/scale.hpp"
#include "scale/std_variant.hpp"
Expand All @@ -31,12 +32,13 @@ namespace kagome::parachain {
const application::AppConfiguration &app_config);

struct PvfWorkerInputConfig {
SCALE_TIE(4);
SCALE_TIE(5);

RuntimeEngine engine;
std::string cache_dir;
std::vector<std::string> log_params;
bool force_disable_secure_mode;
SecureModeSupport secure_mode_support;
};

struct PvfWorkerInputCodeParams {
Expand Down
5 changes: 5 additions & 0 deletions core/parachain/pvf/secure_mode.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ namespace kagome::parachain {

std::string message_;
};

inline auto make_exception_ptr(SecureModeError e) {
return std::make_exception_ptr(std::move(e));
}

template <typename R>
using SecureModeOutcome = CustomOutcome<R, SecureModeError>;

Expand Down
13 changes: 11 additions & 2 deletions core/parachain/pvf/secure_mode_precheck.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include "common/buffer_view.hpp"
#include "log/configurator.hpp"
#include "log/logger.hpp"
#include "parachain/pvf/clone.hpp"
#include "parachain/pvf/secure_mode.hpp"
#include "utils/get_exe_path.hpp"

Expand All @@ -36,8 +37,17 @@ namespace kagome::parachain {
std::filesystem::path cache_dir = original_cache_dir;
SecureModeSupport support = SecureModeSupport::none();
auto logger = log::createLogger("CheckSecureMode", "parachain");
if (auto res = clone::check()) {
support.can_do_secure_clone = true;
} else {
SL_WARN(logger,
"Secure mode incomplete, cannot enable clone for PVF "
"worker: {}",
res.error());
}
if (auto res = changeRoot(cache_dir)) {
support.chroot = true;
cache_dir = "/";
} else {
SL_WARN(
logger,
Expand All @@ -46,7 +56,6 @@ namespace kagome::parachain {
cache_dir.c_str(),
res.error());
}
cache_dir = "/";

if (auto res = enableLandlock(cache_dir)) {
support.landlock = true;
Expand All @@ -70,8 +79,8 @@ namespace kagome::parachain {
}

SecureModeOutcome<SecureModeSupport> runSecureModeCheckProcess(
boost::asio::io_context &io_context,
const std::filesystem::path &cache_dir) {
boost::asio::io_context io_context;
namespace process_v2 = boost::process::v2;
boost::asio::readable_pipe pipe{io_context};
// input passed as CLI arguments to enable users to manually run the check
Expand Down
14 changes: 10 additions & 4 deletions core/parachain/pvf/secure_mode_precheck.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

#pragma once

#include <boost/asio/io_context.hpp>
#include <filesystem>

#include "parachain/pvf/secure_mode.hpp"
Expand All @@ -20,7 +19,7 @@ namespace kagome::parachain {
* platform
*/
struct SecureModeSupport {
SCALE_TIE(3);
SCALE_TIE(4);

// The filesystem root of the PVF process can be set to the worker directory
bool chroot;
Expand All @@ -32,8 +31,16 @@ namespace kagome::parachain {
// process
bool seccomp;

// Whether we are able to call `clone` with all sandboxing flags.
bool can_do_secure_clone;

static SecureModeSupport none() {
return {false, false, false};
return {
.chroot = false,
.landlock = false,
.seccomp = false,
.can_do_secure_clone = false,
};
}

bool isTotallySupported() const {
Expand All @@ -51,7 +58,6 @@ namespace kagome::parachain {
* Spawns a child process that executes checkSecureMode
*/
SecureModeOutcome<SecureModeSupport> runSecureModeCheckProcess(
boost::asio::io_context &io_context,
const std::filesystem::path &cache_dir);

/**
Expand Down
2 changes: 2 additions & 0 deletions core/parachain/pvf/workers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ namespace kagome::parachain {

PvfWorkers::PvfWorkers(const application::AppConfiguration &app_config,
common::MainThreadPool &main_thread_pool,
SecureModeSupport secure_mode_support,
std::shared_ptr<libp2p::basic::Scheduler> scheduler)
: io_context_{main_thread_pool.io_context()},
main_pool_handler_{main_thread_pool.handlerStarted()},
Expand All @@ -122,6 +123,7 @@ namespace kagome::parachain {
.cache_dir = app_config.runtimeCacheDirPath(),
.log_params = app_config.log(),
.force_disable_secure_mode = app_config.disableSecureMode(),
.secure_mode_support = secure_mode_support,
} {
metrics_registry_->registerGaugeFamily(kMetricQueueSize, "pvf queue size");
std::unordered_map<PvfExecTimeoutKind, std::string> kind_name{
Expand Down
1 change: 1 addition & 0 deletions core/parachain/pvf/workers.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ namespace kagome::parachain {
public:
PvfWorkers(const application::AppConfiguration &app_config,
common::MainThreadPool &main_thread_pool,
SecureModeSupport secure_mode_support,
std::shared_ptr<libp2p::basic::Scheduler> scheduler);

using Cb = std::function<void(outcome::result<Buffer>)>;
Expand Down
Loading

0 comments on commit 1f04f5d

Please sign in to comment.