Skip to content

Commit

Permalink
Feature/trie values column (#2288)
Browse files Browse the repository at this point in the history
* Separate trie nodes and values and fix batch processing

* Mark all operator bool's explicit and fix ValueAndHash operator==

* Fix base rocksdb test not deleting ttl_migrated

* Link in_memory_storage to core_integration_test
  • Loading branch information
Harrm authored Dec 2, 2024
1 parent d6c01fa commit 2a9083b
Show file tree
Hide file tree
Showing 110 changed files with 1,139 additions and 756 deletions.
4 changes: 2 additions & 2 deletions core/application/chain_spec.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ namespace kagome::application {
virtual const std::vector<libp2p::multi::Multiaddress> &bootNodes()
const = 0;

virtual const std::vector<std::pair<std::string, size_t>>
&telemetryEndpoints() const = 0;
virtual const std::vector<std::pair<std::string, size_t>> &
telemetryEndpoints() const = 0;

virtual const std::string &protocolId() const = 0;

Expand Down
18 changes: 14 additions & 4 deletions core/application/impl/kagome_application_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

#include "application/impl/kagome_application_impl.hpp"

#include <cstdlib>
#include <soralog/macro.hpp>
#include <thread>

Expand All @@ -17,6 +18,7 @@
#include "injector/application_injector.hpp"
#include "metrics/metrics.hpp"
#include "parachain/pvf/secure_mode_precheck.hpp"
#include "storage/migrations/migrations.hpp"
#include "telemetry/service.hpp"
#include "utils/watchdog.hpp"

Expand Down Expand Up @@ -70,10 +72,11 @@ namespace kagome::application {
getpid());

auto chain_path = app_config_->chainPath(chain_spec_->id());
auto storage_backend = app_config_->storageBackend()
== AppConfiguration::StorageBackend::RocksDB
? "RocksDB"
: "Unknown";
const char *storage_backend =
app_config_->storageBackend()
== AppConfiguration::StorageBackend::RocksDB
? "RocksDB"
: "Unknown";
logger_->info("Chain path is {}, storage backend is {}",
chain_path.native(),
storage_backend);
Expand Down Expand Up @@ -144,6 +147,13 @@ namespace kagome::application {
"platform. Proceed at your own risk.");
#endif

if (app_config_->enableDbMigration()) {
if (auto res = storage::migrations::runMigrations(injector_); !res) {
SL_ERROR(logger_, "Failed to migrate the database: {}", res.error());
exit(EXIT_FAILURE);
}
}

app_state_manager->run();

watchdog->stop();
Expand Down
4 changes: 2 additions & 2 deletions core/blockchain/indexer.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ namespace kagome::blockchain {
*/
template <typename T>
struct Indexer {
Indexer(std::shared_ptr<storage::BufferStorage> db,
Indexer(std::shared_ptr<storage::BufferBatchableStorage> db,
std::shared_ptr<blockchain::BlockTree> block_tree)
: db_{std::move(db)}, block_tree_{std::move(block_tree)} {
primitives::BlockInfo genesis{0, block_tree_->getGenesisBlockHash()};
Expand Down Expand Up @@ -252,7 +252,7 @@ namespace kagome::blockchain {
return raw->kv;
}

std::shared_ptr<storage::BufferStorage> db_;
std::shared_ptr<storage::BufferBatchableStorage> db_;
std::shared_ptr<blockchain::BlockTree> block_tree_;
primitives::BlockInfo last_finalized_indexed_;
std::map<primitives::BlockInfo, Indexed<T>> map_;
Expand Down
2 changes: 1 addition & 1 deletion core/common/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ target_link_libraries(hexutil Boost::boost outcome)
kagome_install(hexutil)

add_library(blob blob.hpp blob.cpp)
target_link_libraries(blob hexutil)
target_link_libraries(blob hexutil scale::scale)
kagome_install(blob)

add_library(fd_limit fd_limit.hpp fd_limit.cpp)
Expand Down
8 changes: 4 additions & 4 deletions core/common/blob.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@
struct fmt::formatter<space_name::class_name> \
: fmt::formatter<space_name::class_name::Base> { \
template <typename FormatCtx> \
auto format(const space_name::class_name &blob, FormatCtx &ctx) const \
-> decltype(ctx.out()) { \
auto format(const space_name::class_name &blob, \
FormatCtx &ctx) const -> decltype(ctx.out()) { \
return fmt::formatter<space_name::class_name::Base>::format(blob, ctx); \
} \
};
Expand Down Expand Up @@ -277,8 +277,8 @@ struct fmt::formatter<kagome::common::Blob<N>> {
// Formats the Blob using the parsed format specification (presentation)
// stored in this formatter.
template <typename FormatContext>
auto format(const kagome::common::Blob<N> &blob, FormatContext &ctx) const
-> decltype(ctx.out()) {
auto format(const kagome::common::Blob<N> &blob,
FormatContext &ctx) const -> decltype(ctx.out()) {
if (presentation == 's') {
if constexpr (N > 4) {
uint16_t head = static_cast<uint16_t>(blob[1])
Expand Down
2 changes: 1 addition & 1 deletion core/consensus/babe/has_babe_consensus_digest.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ namespace kagome::consensus::babe {
}
}

operator bool() const {
explicit operator bool() const {
return epoch.has_value();
}

Expand Down
2 changes: 1 addition & 1 deletion core/consensus/babe/impl/babe_config_repository_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ namespace kagome::consensus::babe {

void warp(Indexer &indexer_, const primitives::BlockInfo &block);

std::shared_ptr<storage::BufferStorage> persistent_storage_;
std::shared_ptr<storage::BufferBatchableStorage> persistent_storage_;
bool config_warp_sync_;
EpochTimings &timings_;
std::shared_ptr<blockchain::BlockTree> block_tree_;
Expand Down
2 changes: 1 addition & 1 deletion core/consensus/grandpa/has_authority_set_change.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ namespace kagome::consensus::grandpa {
}
}

operator bool() const {
explicit operator bool() const {
return scheduled || forced;
}

Expand Down
3 changes: 2 additions & 1 deletion core/consensus/grandpa/impl/authority_manager_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "consensus/grandpa/has_authority_set_change.hpp"
#include "log/logger.hpp"
#include "primitives/event_types.hpp"
#include "storage/buffer_map_types.hpp"
#include "storage/spaced_storage.hpp"

namespace kagome::application {
Expand Down Expand Up @@ -95,7 +96,7 @@ namespace kagome::consensus::grandpa {

std::shared_ptr<blockchain::BlockTree> block_tree_;
std::shared_ptr<runtime::GrandpaApi> grandpa_api_;
std::shared_ptr<storage::BufferStorage> persistent_storage_;
std::shared_ptr<storage::BufferBatchableStorage> persistent_storage_;
primitives::events::ChainSub chain_sub_;

mutable blockchain::Indexer<GrandpaIndexedValue> indexer_;
Expand Down
10 changes: 6 additions & 4 deletions core/consensus/timeline/types.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ namespace kagome::consensus {
template <typename Rep, typename Period>
SlotDuration(const std::chrono::duration<Rep, Period> &duration)
: std::chrono::milliseconds(
std::chrono::duration_cast<std::chrono::milliseconds>(duration)) {}
std::chrono::duration_cast<std::chrono::milliseconds>(duration)) {
}

template <class Rep>
requires std::is_integral_v<Rep>
Expand All @@ -47,7 +48,7 @@ namespace kagome::consensus {
}

// Convert to boolean
operator bool() const {
explicit operator bool() const {
return count() != 0;
}

Expand Down Expand Up @@ -95,8 +96,9 @@ namespace kagome::consensus {
/// Epoch length in slots
EpochLength epoch_length{0};

operator bool() const {
return (bool)slot_duration and (bool) epoch_length;
explicit operator bool() const {
return static_cast<bool>(slot_duration)
and static_cast<bool>(epoch_length);
}

void init(SlotDuration _slot_duration, EpochLength _epoch_length) {
Expand Down
12 changes: 6 additions & 6 deletions core/dispute_coordinator/impl/dispute_coordinator_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1232,25 +1232,25 @@ namespace kagome::dispute {
auto is_old_concluded_for =
intermediate_result.old_state.dispute_status.has_value()
? is_type<ConcludedFor>(
intermediate_result.old_state.dispute_status.value())
intermediate_result.old_state.dispute_status.value())
: false;
auto is_new_concluded_for =
intermediate_result.new_state.dispute_status.has_value()
? is_type<ConcludedFor>(
intermediate_result.new_state.dispute_status.value())
intermediate_result.new_state.dispute_status.value())
: false;
auto is_freshly_concluded_for =
not is_old_concluded_for and is_new_concluded_for;

auto is_old_concluded_against =
intermediate_result.old_state.dispute_status.has_value()
? is_type<ConcludedAgainst>(
intermediate_result.old_state.dispute_status.value())
intermediate_result.old_state.dispute_status.value())
: false;
auto is_new_concluded_against =
intermediate_result.new_state.dispute_status.has_value()
? is_type<ConcludedAgainst>(
intermediate_result.new_state.dispute_status.value())
intermediate_result.new_state.dispute_status.value())
: false;
auto is_freshly_concluded_against =
not is_old_concluded_against and is_new_concluded_against;
Expand All @@ -1261,12 +1261,12 @@ namespace kagome::dispute {
auto is_old_confirmed_concluded =
intermediate_result.old_state.dispute_status.has_value()
? not is_type<Active>(
intermediate_result.old_state.dispute_status.value())
intermediate_result.old_state.dispute_status.value())
: false;
auto is_new_confirmed_concluded =
intermediate_result.new_state.dispute_status.has_value()
? not is_type<Active>(
intermediate_result.new_state.dispute_status.value())
intermediate_result.new_state.dispute_status.value())
: false;
auto is_freshly_confirmed =
not is_old_confirmed_concluded and is_new_confirmed_concluded;
Expand Down
7 changes: 4 additions & 3 deletions core/host_api/host_api.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -411,9 +411,10 @@ namespace kagome::host_api {
* store.
*/
[[nodiscard]] virtual runtime::WasmSpan
ext_crypto_ecdsa_sign_prehashed_version_1(runtime::WasmSize key_type,
runtime::WasmPointer key,
runtime::WasmPointer msg_data) = 0;
ext_crypto_ecdsa_sign_prehashed_version_1(
runtime::WasmSize key_type,
runtime::WasmPointer key,
runtime::WasmPointer msg_data) = 0;

/**
* @brief Generates an ecdsa key for the given key type using an optional
Expand Down
8 changes: 7 additions & 1 deletion core/injector/application_injector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -939,7 +939,7 @@ namespace kagome::injector {
KagomeNodeInjector::KagomeNodeInjector(
sptr<application::AppConfiguration> app_config)
: pimpl_{std::make_unique<KagomeNodeInjectorImpl>(
makeKagomeNodeInjector(std::move(app_config)))} {}
makeKagomeNodeInjector(std::move(app_config)))} {}

sptr<application::AppConfiguration> KagomeNodeInjector::injectAppConfig() {
return pimpl_->injector_
Expand Down Expand Up @@ -1126,6 +1126,12 @@ namespace kagome::injector {
return pimpl_->injector_.template create<sptr<common::MainThreadPool>>();
}

std::shared_ptr<runtime::RuntimeUpgradeTracker>
KagomeNodeInjector::injectRuntimeUpgradeTracker() {
return pimpl_->injector_
.template create<sptr<runtime::RuntimeUpgradeTracker>>();
}

void KagomeNodeInjector::kademliaRandomWalk() {
pimpl_->injector_.create<sptr<KademliaRandomWalk>>();
}
Expand Down
3 changes: 3 additions & 0 deletions core/injector/application_injector.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

#include "clock/clock.hpp"
#include "network/dispute_request_observer.hpp"
#include "runtime/runtime_upgrade_tracker.hpp"
#include "storage/spaced_storage.hpp"

namespace soralog {
Expand Down Expand Up @@ -152,6 +153,8 @@ namespace kagome::injector {
std::shared_ptr<storage::SpacedStorage> injectStorage();
std::shared_ptr<authority_discovery::AddressPublisher>
injectAddressPublisher();
std::shared_ptr<runtime::RuntimeUpgradeTracker>
injectRuntimeUpgradeTracker();
void kademliaRandomWalk();

std::shared_ptr<application::mode::PrintChainInfoMode>
Expand Down
4 changes: 3 additions & 1 deletion core/network/impl/protocols/beefy_protocol_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ namespace kagome::network {
Roles roles,
std::shared_ptr<Beefy> beefy)
: notifications_{notifications_factory.make(
{make_protocols(kBeefyProtocol, genesis)}, kPeersLimit, kPeersLimit)},
{make_protocols(kBeefyProtocol, genesis)},
kPeersLimit,
kPeersLimit)},
roles_{roles},
beefy_{std::move(beefy)} {}

Expand Down
22 changes: 11 additions & 11 deletions core/network/impl/protocols/fetch_attested_candidate.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,17 @@ namespace kagome::network {
parachain::statement_distribution::StatementDistribution>
statement_distribution)
: RequestResponseProtocolImpl<
vstaging::AttestedCandidateRequest,
vstaging::AttestedCandidateResponse,
ScaleMessageReadWriter>{kFetchAttestedCandidateProtocolName,
host,
make_protocols(
kFetchAttestedCandidateProtocol,
genesis_hash,
kProtocolPrefixPolkadot),
log::createLogger(
kFetchAttestedCandidateProtocolName,
"req_attested_candidate_protocol")},
vstaging::AttestedCandidateRequest,
vstaging::AttestedCandidateResponse,
ScaleMessageReadWriter>{kFetchAttestedCandidateProtocolName,
host,
make_protocols(
kFetchAttestedCandidateProtocol,
genesis_hash,
kProtocolPrefixPolkadot),
log::createLogger(
kFetchAttestedCandidateProtocolName,
"req_attested_candidate_protocol")},
statement_distribution_(std::move(statement_distribution)) {
BOOST_ASSERT(statement_distribution_);
}
Expand Down
2 changes: 1 addition & 1 deletion core/network/impl/protocols/parachain.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ namespace kagome::network {
size_t limit_in,
size_t limit_out)
: notifications_{inject.notifications_factory->make(
std::move(protocols_groups), limit_in, limit_out)},
std::move(protocols_groups), limit_in, limit_out)},
collation_versions_{CollationVersion::VStaging, CollationVersion::V1},
roles_{inject.roles},
peer_manager_{inject.peer_manager},
Expand Down
18 changes: 9 additions & 9 deletions core/network/impl/protocols/protocol_fetch_chunk.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,15 @@ namespace kagome::network {
std::shared_ptr<parachain::ParachainProcessorImpl> pp,
std::shared_ptr<PeerManager> pm)
: RequestResponseProtocolImpl<
FetchChunkRequest,
FetchChunkResponse,
ScaleMessageReadWriter>{kFetchChunkProtocolName,
host,
make_protocols(kFetchChunkProtocol,
genesis_hash,
kProtocolPrefixPolkadot),
log::createLogger(kFetchChunkProtocolName,
"req_chunk_protocol")},
FetchChunkRequest,
FetchChunkResponse,
ScaleMessageReadWriter>{kFetchChunkProtocolName,
host,
make_protocols(kFetchChunkProtocol,
genesis_hash,
kProtocolPrefixPolkadot),
log::createLogger(kFetchChunkProtocolName,
"req_chunk_protocol")},
pp_{std::move(pp)},
pm_{std::move(pm)} {
BOOST_ASSERT(pp_);
Expand Down
20 changes: 11 additions & 9 deletions core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "blockchain/genesis_block_hash.hpp"
#include "log/logger.hpp"
#include "network/common.hpp"
#include "network/helpers/scale_message_read_writer.hpp"
#include "network/impl/protocols/request_response_protocol.hpp"
#include "parachain/validator/parachain_processor.hpp"
#include "utils/non_copyable.hpp"
Expand Down Expand Up @@ -44,15 +45,16 @@ namespace kagome::network {
const blockchain::GenesisBlockHash &genesis_hash,
std::shared_ptr<parachain::ParachainProcessorImpl> pp)
: RequestResponseProtocolImpl<
FetchChunkRequest,
FetchChunkResponseObsolete,
ScaleMessageReadWriter>{kFetchChunkProtocolName,
host,
make_protocols(kFetchChunkProtocolObsolete,
genesis_hash,
kProtocolPrefixPolkadot),
log::createLogger(kFetchChunkProtocolName,
"req_chunk_protocol")},
FetchChunkRequest,
FetchChunkResponseObsolete,
ScaleMessageReadWriter>{kFetchChunkProtocolName,
host,
make_protocols(
kFetchChunkProtocolObsolete,
genesis_hash,
kProtocolPrefixPolkadot),
log::createLogger(kFetchChunkProtocolName,
"req_chunk_protocol")},
pp_{std::move(pp)} {
BOOST_ASSERT(pp_);
}
Expand Down
6 changes: 3 additions & 3 deletions core/network/impl/protocols/protocol_req_collation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,9 @@ namespace kagome::network {
const blockchain::GenesisBlockHash &genesis_hash,
std::shared_ptr<ReqCollationObserver> observer)
: v1_impl_{std::make_shared<
ReqCollationProtocolImpl<CollationFetchingRequest,
CollationFetchingResponse>>(
host, kReqCollationProtocol, chain_spec, genesis_hash, observer)},
ReqCollationProtocolImpl<CollationFetchingRequest,
CollationFetchingResponse>>(
host, kReqCollationProtocol, chain_spec, genesis_hash, observer)},
vstaging_impl_{std::make_shared<
ReqCollationProtocolImpl<vstaging::CollationFetchingRequest,
vstaging::CollationFetchingResponse>>(
Expand Down
Loading

0 comments on commit 2a9083b

Please sign in to comment.