Skip to content

Commit

Permalink
max_parallel_downloads are passed by cli
Browse files Browse the repository at this point in the history
  • Loading branch information
ErakhtinB committed Dec 20, 2024
1 parent 2d1c3a5 commit f564980
Show file tree
Hide file tree
Showing 6 changed files with 31 additions and 12 deletions.
4 changes: 4 additions & 0 deletions core/application/app_configuration.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,10 @@ namespace kagome::application {
const = 0;

virtual std::optional<PrecompileWasmConfig> precompileWasm() const = 0;
/**
* @return maximum number of parallel downloads of announced block per peer
*/
virtual uint32_t maxParallelDownloads() const = 0;
};

} // namespace kagome::application
19 changes: 13 additions & 6 deletions core/application/impl/app_configuration_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ namespace {
#endif
const uint32_t def_db_cache_size = 1024;
const uint32_t def_parachain_runtime_instance_cache_size = 100;
constexpr auto max_concurrent_block_announce_validations_per_peer = 5;

/**
* Generate once at run random node name if form of UUID
Expand Down Expand Up @@ -176,12 +177,11 @@ namespace {

static constexpr std::array<std::string_view,
1 + KAGOME_WASM_COMPILER_WASM_EDGE>
interpreters {
interpreters{
#if KAGOME_WASM_COMPILER_WASM_EDGE == 1
"WasmEdge",
"WasmEdge",
#endif
"Binaryen"
};
"Binaryen"};

static const std::string interpreters_str =
fmt::format("[{}]", fmt::join(interpreters, ", "));
Expand Down Expand Up @@ -841,6 +841,9 @@ namespace kagome::application {
("rpc-methods", po::value<std::string>(), R"("auto" (default), "unsafe", "safe")")
("no-mdns", po::bool_switch(), "(unused, zombienet stub)")
("prometheus-external", po::bool_switch(), "alias for \"--prometheus-host 0.0.0.0\"")
("max-parallel-downloads", po::value<uint32_t>()->default_value(max_concurrent_block_announce_validations_per_peer),
"Maximum number of peers from which to ask for the same blocks in parallel."
"This allows downloading announced blocks from multiple peers. Decrease to save traffic and risk increased latency.")
;

po::options_description development_desc("Additional options");
Expand Down Expand Up @@ -912,8 +915,8 @@ namespace kagome::application {
}

if (vm.count("help") > 0) {
std::cout
<< "Available subcommands: storage-explorer db-editor benchmark key\n";
std::cout << "Available subcommands: storage-explorer db-editor "
"benchmark key\n";
std::cout << desc << '\n';
return false;
}
Expand Down Expand Up @@ -1600,6 +1603,10 @@ namespace kagome::application {
runtime_exec_method_ = RuntimeExecutionMethod::Compile;
}

max_parallel_downloads_ =
find_argument<uint32_t>(vm, "max-parallel-downloads")
.value_or(max_concurrent_block_announce_validations_per_peer);

// if something wrong with config print help message
if (not validate_config()) {
std::cout << desc << '\n';
Expand Down
4 changes: 4 additions & 0 deletions core/application/impl/app_configuration_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,9 @@ namespace kagome::application {
std::optional<PrecompileWasmConfig> precompileWasm() const override {
return precompile_wasm_;
}
uint32_t maxParallelDownloads() const override {
return max_parallel_downloads_;
}

private:
void parse_general_segment(const rapidjson::Value &val);
Expand Down Expand Up @@ -368,6 +371,7 @@ namespace kagome::application {
bool prune_discarded_states_ = false;
bool enable_thorough_pruning_ = false;
std::optional<uint32_t> blocks_pruning_;
uint32_t max_parallel_downloads_;
bool enable_db_migration_ = false;
std::optional<std::string> dev_mnemonic_phrase_;
std::string node_wss_pem_;
Expand Down
10 changes: 4 additions & 6 deletions core/network/impl/protocols/block_announce_protocol.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ namespace kagome::network {
// https://github.com/paritytech/polkadot-sdk/blob/edf79aa972bcf2e043e18065a9bb860ecdbd1a6e/substrate/client/network/sync/src/engine.rs#L86
constexpr size_t kSeenCapacity = 1024;

constexpr auto MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER = 4;

static const struct {
void inc(bool inc) const {
for (auto &metric : metrics) {
Expand Down Expand Up @@ -77,7 +75,8 @@ namespace kagome::network {
hasher_(std::move(hasher)),
telemetry_peer_count_(std::move(telemetry_peer_count)),
peer_manager_{std::move(peer_manager)},
seen_{kSeenCapacity} {
seen_{kSeenCapacity},
max_parallel_downloads_{app_config.maxParallelDownloads()} {
BOOST_ASSERT(block_tree_ != nullptr);
BOOST_ASSERT(observer_ != nullptr);
BOOST_ASSERT(peer_manager_ != nullptr);
Expand Down Expand Up @@ -128,8 +127,7 @@ namespace kagome::network {
{
std::shared_lock lock(active_peers_mutex_);
selected_peers.push_back(peer_id);
if (active_peers_.size()
> MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER) {
if (active_peers_.size() > max_parallel_downloads_) {
std::vector<PeerId> temp_peers;
std::copy_if(active_peers_.begin(),
active_peers_.end(),
Expand All @@ -138,7 +136,7 @@ namespace kagome::network {
std::sample(temp_peers.begin(),
temp_peers.end(),
std::back_inserter(selected_peers),
MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER - 1,
max_parallel_downloads_ ? max_parallel_downloads_ - 1 : 0,
std::mt19937{std::random_device{}()});
} else {
std::copy_if(active_peers_.begin(),
Expand Down
1 change: 1 addition & 0 deletions core/network/impl/protocols/block_announce_protocol.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ namespace kagome::network {
MapLruSet<PeerId, Hash256> seen_;
std::shared_mutex active_peers_mutex_;
std::unordered_set<PeerId> active_peers_;
uint32_t max_parallel_downloads_;
};

} // namespace kagome::network
5 changes: 5 additions & 0 deletions test/mock/core/application/app_configuration_mock.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,11 @@ namespace kagome::application {
precompileWasm,
(),
(const, override));

MOCK_METHOD(std::optional<uint32_t>,
maxParallelDownloads,
(),
(const, override));
};

} // namespace kagome::application

0 comments on commit f564980

Please sign in to comment.