diff --git a/core/authority_discovery/query/authority_peer_info.hpp b/core/authority_discovery/query/authority_peer_info.hpp index 056da258ad..5393aa6d11 100644 --- a/core/authority_discovery/query/authority_peer_info.hpp +++ b/core/authority_discovery/query/authority_peer_info.hpp @@ -18,7 +18,7 @@ namespace kagome::authority_discovery { common::Buffer raw{}; std::optional time{}; - scale::PeerInfoSerializable peer{}; + ::scale::PeerInfoSerializable peer{}; }; } // namespace kagome::authority_discovery diff --git a/core/authority_discovery/timestamp.hpp b/core/authority_discovery/timestamp.hpp index f9cae7e204..d62d9cb057 100644 --- a/core/authority_discovery/timestamp.hpp +++ b/core/authority_discovery/timestamp.hpp @@ -9,6 +9,6 @@ #include "scale/big_fixed_integers.hpp" namespace kagome::authority_discovery { - using Timestamp = scale::uint128_t; - using TimestampScale = scale::Fixed; + using Timestamp = ::scale::uint128_t; + using TimestampScale = ::scale::Fixed; } // namespace kagome::authority_discovery diff --git a/core/crypto/type_hasher.hpp b/core/crypto/type_hasher.hpp index a3430e6055..96fa776772 100644 --- a/core/crypto/type_hasher.hpp +++ b/core/crypto/type_hasher.hpp @@ -15,8 +15,11 @@ namespace kagome::crypto { template inline void hashTypes(H &hasher, common::Blob &out, T &&...t) { - auto val = ::scale::encode(std::forward(t)...).value(); - hasher.update(val); + scale::encode( + [&](const uint8_t *const ptr, size_t count) { + hasher.update(std::span(ptr, count)); + }, + std::forward(t)...); hasher.get_final(out); } diff --git a/core/network/helpers/scale_message_read_writer.hpp b/core/network/helpers/scale_message_read_writer.hpp index d605d81848..8432a580e8 100644 --- a/core/network/helpers/scale_message_read_writer.hpp +++ b/core/network/helpers/scale_message_read_writer.hpp @@ -66,7 +66,7 @@ namespace kagome::network { template void write(const MsgType &msg, libp2p::basic::Writer::WriteCallbackFunc cb) const { - auto encoded_msg_res = ::scale::encode(msg); + auto encoded_msg_res = scale::encode(msg); if (!encoded_msg_res) { return cb(encoded_msg_res.error()); } diff --git a/core/network/impl/sync_protocol_observer_impl.cpp b/core/network/impl/sync_protocol_observer_impl.cpp index 50b802bf97..1eddcd0b7f 100644 --- a/core/network/impl/sync_protocol_observer_impl.cpp +++ b/core/network/impl/sync_protocol_observer_impl.cpp @@ -196,7 +196,7 @@ namespace kagome::network { if (auto r = beefy_->getJustification(*number)) { if (auto &opt = r.value()) { new_block.beefy_justification = primitives::Justification{ - common::Buffer{::scale::encode(*opt).value()}, + common::Buffer{scale::encode(*opt).value()}, }; } } diff --git a/core/network/types/collator_messages.hpp b/core/network/types/collator_messages.hpp index 0687abd3cd..625a9cdf7e 100644 --- a/core/network/types/collator_messages.hpp +++ b/core/network/types/collator_messages.hpp @@ -458,8 +458,7 @@ namespace kagome::network { auto commitments_hash = hasher.blake2b_256(scale::encode(receipt.commitments).value()); return hasher.blake2b_256( - ::scale::encode(std::tie(receipt.descriptor, commitments_hash)) - .value()); + scale::encode(std::tie(receipt.descriptor, commitments_hash)).value()); } inline CandidateHash candidateHash(const crypto::Hasher &hasher, diff --git a/core/parachain/approval/knowledge.hpp b/core/parachain/approval/knowledge.hpp index 9d6ff08890..5cb51df094 100644 --- a/core/parachain/approval/knowledge.hpp +++ b/core/parachain/approval/knowledge.hpp @@ -11,13 +11,14 @@ #include "common/visitor.hpp" #include "consensus/timeline/types.hpp" #include "outcome/outcome.hpp" +#include "parachain/approval/approval.hpp" #include "parachain/approval/state.hpp" #include "parachain/types.hpp" template <> struct std::hash { auto operator()(const scale::BitVec &v) const { - auto s = ::scale::encode(v).value(); + auto s = scale::encode(v).value(); return boost::hash_range(s.begin(), s.end()); } }; @@ -86,7 +87,7 @@ namespace kagome::parachain::approval { // Generate the knowledge keys for querying if an approval is known by peer. static std::pair generate_approval_key( - const approval::IndirectSignedApprovalVoteV2 &approval) { + const IndirectSignedApprovalVoteV2 &approval) { return { std::make_tuple(approval.payload.payload.block_hash, approval.payload.payload.candidate_indices, diff --git a/core/parachain/pvf/pvf_impl.cpp b/core/parachain/pvf/pvf_impl.cpp index 327a8128c6..cdec26202a 100644 --- a/core/parachain/pvf/pvf_impl.cpp +++ b/core/parachain/pvf/pvf_impl.cpp @@ -306,7 +306,7 @@ namespace kagome::parachain { receipt.descriptor.relay_parent, receipt.descriptor.para_id); - auto data_hash = hasher_->blake2b_256(::scale::encode(pvd).value()); + auto data_hash = hasher_->blake2b_256(scale::encode(pvd).value()); if (receipt.descriptor.persisted_data_hash != data_hash) { return cb(PvfError::PERSISTED_DATA_HASH); } @@ -391,6 +391,7 @@ namespace kagome::parachain { } cb(scale::decode(r.value())); }, + .kind = timeout_kind, .timeout = std::chrono::milliseconds{ timeout_kind == runtime::PvfExecTimeoutKind::Backing diff --git a/core/parachain/pvf/workers.cpp b/core/parachain/pvf/workers.cpp index 617ed76a6a..b05c0167b9 100644 --- a/core/parachain/pvf/workers.cpp +++ b/core/parachain/pvf/workers.cpp @@ -20,6 +20,8 @@ #include "utils/weak_macro.hpp" namespace kagome::parachain { + constexpr auto kMetricQueueSize = "kagome_pvf_queue_size"; + struct AsyncPipe : boost::process::async_pipe { using async_pipe::async_pipe; using lowest_layer_type = AsyncPipe; @@ -135,13 +137,26 @@ namespace kagome::parachain { .log_params = app_config.log(), .force_disable_secure_mode = app_config.disableSecureMode(), .secure_mode_support = secure_mode_support, - } {} + } { + metrics_registry_->registerGaugeFamily(kMetricQueueSize, "pvf queue size"); + std::unordered_map kind_name{ + {PvfExecTimeoutKind::Approval, "Approval"}, + {PvfExecTimeoutKind::Backing, "Backing"}, + }; + for (auto &[kind, name] : kind_name) { + metric_queue_size_.emplace(kind, + metrics_registry_->registerGaugeMetric( + kMetricQueueSize, {{"kind", name}})); + } + } void PvfWorkers::execute(Job &&job) { REINVOKE(*main_pool_handler_, execute, std::move(job)); if (free_.empty()) { if (used_ >= max_) { - queue_.emplace(std::move(job)); + auto &queue = queues_[job.kind]; + queue.emplace_back(std::move(job)); + metric_queue_size_.at(job.kind)->set(queue.size()); return; } auto used = std::make_shared(*this); @@ -245,11 +260,16 @@ namespace kagome::parachain { } void PvfWorkers::dequeue() { - if (queue_.empty()) { - return; + for (auto &kind : + {PvfExecTimeoutKind::Approval, PvfExecTimeoutKind::Backing}) { + auto &queue = queues_[kind]; + if (queue.empty()) { + continue; + } + auto job = std::move(queue.front()); + queue.pop_front(); + metric_queue_size_.at(kind)->set(queue.size()); + findFree(std::move(job)); } - auto job = std::move(queue_.front()); - queue_.pop(); - findFree(std::move(job)); } } // namespace kagome::parachain diff --git a/core/parachain/pvf/workers.hpp b/core/parachain/pvf/workers.hpp index ed6d362ae6..d7d4ecd837 100644 --- a/core/parachain/pvf/workers.hpp +++ b/core/parachain/pvf/workers.hpp @@ -6,11 +6,13 @@ #pragma once +#include #include #include -#include +#include "metrics/metrics.hpp" #include "parachain/pvf/pvf_worker_types.hpp" +#include "runtime/runtime_api/parachain_host_types.hpp" namespace boost::asio { class io_context; @@ -33,6 +35,8 @@ namespace kagome::common { } // namespace kagome::common namespace kagome::parachain { + using runtime::PvfExecTimeoutKind; + struct ProcessAndPipes; class PvfWorkers : public std::enable_shared_from_this { @@ -47,6 +51,7 @@ namespace kagome::parachain { PvfWorkerInputCodeParams code_params; Buffer args; Cb cb; + PvfExecTimeoutKind kind; std::chrono::milliseconds timeout{0}; }; void execute(Job &&job); @@ -78,6 +83,9 @@ namespace kagome::parachain { PvfWorkerInputConfig worker_config_; std::list free_; size_t used_ = 0; - std::queue queue_; + std::unordered_map> queues_; + + metrics::RegistryPtr metrics_registry_ = metrics::createRegistry(); + std::unordered_map metric_queue_size_; }; } // namespace kagome::parachain diff --git a/core/parachain/types.hpp b/core/parachain/types.hpp index b2ea0e21e0..5d2c26ee4d 100644 --- a/core/parachain/types.hpp +++ b/core/parachain/types.hpp @@ -93,7 +93,7 @@ namespace kagome::parachain { auto signable() { constexpr std::array kMagic{'V', 'C', 'P', 'C'}; - return ::scale::encode(std::make_tuple(kMagic, *this)).value(); + return scale::encode(std::make_tuple(kMagic, *this)).value(); } }; } // namespace kagome::parachain @@ -171,11 +171,11 @@ namespace kagome::network { common::Buffer signable() const { return common::Buffer{ - ::scale::encode(relay_parent, - para_id, - persisted_data_hash, - pov_hash, - validation_code_hash) + scale::encode(relay_parent, + para_id, + persisted_data_hash, + pov_hash, + validation_code_hash) .value(), }; } @@ -193,7 +193,7 @@ namespace kagome::network { const parachain::Hash &hash(const crypto::Hasher &hasher) const { if (not hash_.has_value()) { hash_.emplace(hasher.blake2b_256( - ::scale::encode(std::tie(descriptor, commitments_hash)).value())); + scale::encode(std::tie(descriptor, commitments_hash)).value())); } return hash_.value(); } diff --git a/core/parachain/validator/signer.hpp b/core/parachain/validator/signer.hpp index 2151f0dce1..d264173e90 100644 --- a/core/parachain/validator/signer.hpp +++ b/core/parachain/validator/signer.hpp @@ -48,7 +48,7 @@ namespace kagome::parachain { template auto signable(const crypto::Hasher &hasher, const T &payload) const { auto &&signable = toSignable(hasher, payload); - return ::scale::encode(std::tie(signable, *this)).value(); + return scale::encode(std::tie(signable, *this)).value(); } /// Current session index. diff --git a/core/parachain/validator/statement_distribution/types.hpp b/core/parachain/validator/statement_distribution/types.hpp index fb20cf0367..ef4c890365 100644 --- a/core/parachain/validator/statement_distribution/types.hpp +++ b/core/parachain/validator/statement_distribution/types.hpp @@ -33,7 +33,7 @@ namespace kagome::parachain { statement, [&](const StatementWithPVDSeconded &val) { return hasher->blake2b_256( - ::scale::encode(val.committed_receipt.to_plain(*hasher)).value()); + scale::encode(val.committed_receipt.to_plain(*hasher)).value()); }, [&](const StatementWithPVDValid &val) { return val.candidate_hash; }); } diff --git a/core/primitives/inherent_data.hpp b/core/primitives/inherent_data.hpp index 5431f19bf4..2483ec94e0 100644 --- a/core/primitives/inherent_data.hpp +++ b/core/primitives/inherent_data.hpp @@ -50,7 +50,7 @@ namespace kagome::primitives { const T &inherent) { auto [it, inserted] = data.try_emplace(identifier, common::Buffer()); if (inserted) { - it->second = common::Buffer(::scale::encode(inherent).value()); + it->second = common::Buffer(scale::encode(inherent).value()); return outcome::success(); } return InherentDataError::IDENTIFIER_ALREADY_EXISTS; @@ -62,7 +62,7 @@ namespace kagome::primitives { */ template void replaceData(InherentIdentifier identifier, const T &inherent) { - data[identifier] = common::Buffer(::scale::encode(inherent).value()); + data[identifier] = common::Buffer(scale::encode(inherent).value()); } /** diff --git a/core/runtime/module_instance.hpp b/core/runtime/module_instance.hpp index 49e5d244db..1c2c10a26d 100644 --- a/core/runtime/module_instance.hpp +++ b/core/runtime/module_instance.hpp @@ -18,6 +18,7 @@ #include "outcome/outcome.hpp" #include "runtime/instance_environment.hpp" #include "runtime/ptr_size.hpp" +#include "scale/kagome_scale.hpp" namespace kagome::runtime { class Module; @@ -43,9 +44,9 @@ namespace kagome::runtime { template static outcome::result encodeArgs(const Args &...args) { if constexpr (sizeof...(args) > 0) { - return common::map_result(::scale::encode(args...), [](auto &&vec) { - return common::Buffer{vec}; - }); + return common::map_result( + kagome::scale::encode(args...), + [](auto &&vec) { return common::Buffer{vec}; }); } return outcome::success(); } diff --git a/core/scale/encoder/concepts.hpp b/core/scale/encoder/concepts.hpp new file mode 100644 index 0000000000..e9d71c298d --- /dev/null +++ b/core/scale/encoder/concepts.hpp @@ -0,0 +1,22 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace kagome::scale { + template + concept Invocable = std::is_invocable_v; + + template + concept IsEnum = std::is_enum_v>; + + template + concept IsNotEnum = ! + std::is_enum_v>; +} // namespace kagome::scale diff --git a/core/scale/encoder/primitives.hpp b/core/scale/encoder/primitives.hpp index eb48622c88..8b09d7619f 100644 --- a/core/scale/encoder/primitives.hpp +++ b/core/scale/encoder/primitives.hpp @@ -7,7 +7,10 @@ #ifndef KAGOME_SCALE_ENCODER_PRIMITIVES_HPP #define KAGOME_SCALE_ENCODER_PRIMITIVES_HPP +#include #include +#include +#include #include #include #include @@ -16,82 +19,90 @@ #include #include #include - +#include "crypto/ecdsa_types.hpp" +#include "primitives/math.hpp" +#include "scale/encoder/concepts.hpp" #include "utils/struct_to_tuple.hpp" namespace kagome::scale { + constexpr void putByte(const Invocable auto &func, + const uint8_t *const val, + size_t count); + + template + constexpr void encode(const Invocable auto &func, const std::tuple &v); - template - constexpr void putByte(const F &func, const uint8_t *const val, size_t count); + template + constexpr void encode(const Invocable auto &func, + const T &t, + const Args &...args); - template - constexpr void encode(const F &func, const std::tuple &v); + template + constexpr void encode(const Invocable auto &func, const std::vector &c); - template - constexpr void encode(const F &func, const T &t, const Args &...args); + template + constexpr void encode(const Invocable auto &func, const std::pair &p); - template - constexpr void encode(const F &func, const std::vector &c); + template + constexpr void encode(const Invocable auto &func, const std::span &c); - template - constexpr void encode(const FN &func, const std::pair &p); + template + constexpr void encode(const Invocable auto &func, const std::span &c); - template - constexpr void encode(const F &func, const std::span &c); + template + constexpr void encode(const Invocable auto &func, + const std::array &c); - template - constexpr void encode(const F &func, const std::span &c); + // NOLINTBEGIN + template + constexpr void encode(const Invocable auto &func, const T (&c)[N]); + // NOLINTEND - template - constexpr void encode(const F &func, const std::array &c); + template + constexpr void encode(const Invocable auto &func, const std::map &c); - template - constexpr void encode(const F &func, const T (&c)[N]); + template + constexpr void encode(const Invocable auto &func, + const std::shared_ptr &v); - template - constexpr void encode(const F &func, const std::map &c); + constexpr void encode(const Invocable auto &func, const std::string_view &v); - template - constexpr void encode(const F &func, const std::shared_ptr &v); + constexpr void encode(const Invocable auto &func, const std::string &v); - template - constexpr void encode(const F &func, const std::string_view &v); + template + constexpr void encode(const Invocable auto &func, + const std::unique_ptr &v); - template - constexpr void encode(const F &func, const std::string &v); + template + constexpr void encode(const Invocable auto &func, const std::list &c); - template - constexpr void encode(const F &func, const std::unique_ptr &v); + template + constexpr void encode(const Invocable auto &func, const std::deque &c); - template - constexpr void encode(const F &func, const std::list &c); + template + void encode(const Invocable auto &func, const boost::variant &v); - template - constexpr void encode(const F &func, const std::deque &c); + template + void encode(const Invocable auto &func, const boost::variant &v); - template - void encode(const F &func, const boost::variant &v); + template + void encode(const Invocable auto &func, const std::variant &v); - template - void encode(const F &func, const boost::variant &v); + void encode(const Invocable auto &func, const ::scale::CompactInteger &value); - template - void encode(const F &func, const ::scale::CompactInteger &value); + void encode(const Invocable auto &func, const ::scale::BitVec &value); - template - void encode(const F &func, const ::scale::BitVec &value); + void encode(const Invocable auto &func, const std::optional &value); - template - void encode(const F &func, const std::optional &value); + template + void encode(const Invocable auto &func, const std::optional &value); - template - void encode(const F &func, const std::optional &value); + void encode(const Invocable auto &func, const crypto::EcdsaSignature &value); - template >, bool> = true> - constexpr void encode(const F &func, const T &v) { - using I = std::decay_t; + void encode(const Invocable auto &func, const crypto::EcdsaPublicKey &value); + + constexpr void encode(const Invocable auto &func, const IsNotEnum auto &v) { + using I = std::decay_t; if constexpr (std::is_integral_v) { if constexpr (std::is_same_v) { const uint8_t byte = (v ? 1u : 0u); @@ -108,26 +119,37 @@ namespace kagome::scale { const auto val = math::toLE(v); putByte(func, (uint8_t *)&val, size); } else { - encode(func, utils::to_tuple_refs(v)); + kagome::scale::encode(func, utils::to_tuple_refs(v)); } } - template >, bool> = true> - constexpr void encode(const F &func, const T &value) { - encode(func, static_cast>>(value)); + constexpr void encode(const Invocable auto &func, const IsEnum auto &value) { + kagome::scale::encode( + func, + static_cast>>( + value)); } - template - constexpr void encode(const F &func, const T &t, const Args &...args) { - encode(func, t); - encode(func, args...); + template + constexpr void encode(const Invocable auto &func, + const T &t, + const Args &...args) { + kagome::scale::encode(func, t); + kagome::scale::encode(func, args...); } + // NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic) template outcome::result> encode(const Args &...args) { - return ::scale::encode(args...); + std::vector res; + kagome::scale::encode( + [&](const uint8_t *const val, size_t count) { + if (count != 0ull) { + res.insert(res.end(), &val[0], &val[count]); + } + }, + args...); + return res; } inline size_t bitUpperBorder(const ::scale::CompactInteger &x) { @@ -168,71 +190,71 @@ namespace kagome::scale { } return counter; } + // NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic) - template - constexpr void putByte(const F &func, + constexpr void putByte(const Invocable auto &func, const uint8_t *const val, size_t count) { func(val, count); } - template - void encode(const F &func, const boost::variant &v) { + template + void encode(const Invocable auto &func, const boost::variant &v) { using T = std::tuple_element_t>; if (v.which() == I) { - encode(func, I); - encode(func, boost::get(v)); + kagome::scale::encode(func, I); + kagome::scale::encode(func, boost::get(v)); return; } if constexpr (sizeof...(Ts) > I + 1) { - encode(func, v); + kagome::scale::encode(func, v); } } - template - constexpr void encode(const F &func, const T (&c)[N]) { + // NOLINTBEGIN + template + constexpr void encode(const Invocable auto &func, const T (&c)[N]) { using E = std::decay_t; if constexpr (std::is_integral_v && sizeof(E) == 1u) { putByte(func, c, N); } else { for (const auto &e : c) { - encode(func, e); + kagome::scale::encode(func, e); } } } + // NOLINTEND - template - void encode(const F &func, const boost::variant &v) { - encode(func, v); + template + void encode(const Invocable auto &func, const boost::variant &v) { + kagome::scale::encode<0>(func, v); } - template , std::enable_if_t, bool> = true> - constexpr void encodeCompactSmall(const F &func, T val) { + constexpr void encodeCompactSmall(const Invocable auto &func, T val) { BOOST_ASSERT_MSG((val >> (8 * sizeof(I) - 2)) == 0, "Unexpected compact value in encoder"); val <<= 2; val |= (sizeof(I) / 2ull); - encode(func, val); + kagome::scale::encode(func, val); } - template - void encodeCompact(const F &func, uint64_t val) { + void encodeCompact(const Invocable auto &func, uint64_t val) { if (val < ::scale::compact::EncodingCategoryLimits::kMinUint16) { - encodeCompactSmall(func, static_cast(val)); + kagome::scale::encodeCompactSmall(func, static_cast(val)); return; } if (val < ::scale::compact::EncodingCategoryLimits::kMinUint32) { - encodeCompactSmall(func, static_cast(val)); + kagome::scale::encodeCompactSmall(func, static_cast(val)); return; } if (val < ::scale::compact::EncodingCategoryLimits::kMinBigInteger) { - encodeCompactSmall(func, static_cast(val)); + kagome::scale::encodeCompactSmall(func, static_cast(val)); return; } @@ -245,24 +267,27 @@ namespace kagome::scale { putByte(func, result, bigIntLength + 1ull); } - template - constexpr void encode(const F &func, const std::string &v) { - encode(func, std::string_view{v}); + template + void encode(const Invocable auto &func, const std::variant &v) { + kagome::scale::encode(func, (uint8_t)v.index()); + std::visit([&](const auto &s) { kagome::scale::encode(func, s); }, v); + } + + constexpr void encode(const Invocable auto &func, const std::string &v) { + kagome::scale::encode(func, std::string_view{v}); } - template - constexpr void encode(const F &func, const std::string_view &v) { - encodeCompact(func, v.size()); + constexpr void encode(const Invocable auto &func, const std::string_view &v) { + kagome::scale::encodeCompact(func, v.size()); putByte(func, (const uint8_t *)v.data(), v.size()); } - template - void encode(const F &func, const ::scale::BitVec &v) { + void encode(const Invocable auto &func, const ::scale::BitVec &v) { const size_t bitsCount = v.bits.size(); const size_t bytesCount = ((bitsCount + 7ull) >> 3ull); const size_t blocksCount = ((bytesCount + 7ull) >> 3ull); - encodeCompact(func, bitsCount); + kagome::scale::encodeCompact(func, bitsCount); uint64_t result; size_t bitCounter = 0ull; for (size_t ix = 0ull; ix < blocksCount; ++ix) { @@ -282,25 +307,25 @@ namespace kagome::scale { } } - template - void encode(const F &func, const ::scale::CompactInteger &value) { + void encode(const Invocable auto &func, + const ::scale::CompactInteger &value) { if (value < 0) { raise(::scale::EncodeError::NEGATIVE_COMPACT_INTEGER); } const size_t bit_border = bitUpperBorder(value); if (bit_border <= 6) { // kMinUint16 - encodeCompactSmall(func, value.convert_to()); + kagome::scale::encodeCompactSmall(func, value.convert_to()); return; } if (bit_border <= 14) { // kMinUint32 - encodeCompactSmall(func, value.convert_to()); + kagome::scale::encodeCompactSmall(func, value.convert_to()); return; } if (bit_border <= 30) { // kMinBigInteger - encodeCompactSmall(func, value.convert_to()); + kagome::scale::encodeCompactSmall(func, value.convert_to()); return; } @@ -329,28 +354,27 @@ namespace kagome::scale { } template < - typename F, typename It, typename = std::enable_if_t< !std::is_same_v::value_type, void>>> - constexpr void encode(const F &func, It begin, It end) { + constexpr void encode(const Invocable auto &func, It begin, It end) { while (begin != end) { - encode(func, *begin); + kagome::scale::encode(func, *begin); ++begin; } } - template - constexpr void encode(const F &func, const std::span &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::span &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const F &func, const std::span &c) { + template + constexpr void encode(const Invocable auto &func, const std::span &c) { if constexpr (S == -1) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } else { using E = std::decay_t; if constexpr (std::is_integral_v && sizeof(E) == 1u) { @@ -363,68 +387,72 @@ namespace kagome::scale { } } - template - constexpr void encode(const F &func, const std::array &c) { + template + constexpr void encode(const Invocable auto &func, + const std::array &c) { for (const auto &e : c) { - encode(func, e); + kagome::scale::encode(func, e); } } - template - constexpr void encode(const F &func, const std::map &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::map &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const FN &func, const std::pair &p) { - encode(func, p.first); - encode(func, p.second); + template + constexpr void encode(const Invocable auto &func, const std::pair &p) { + kagome::scale::encode(func, p.first); + kagome::scale::encode(func, p.second); } - template - constexpr void encode(const F &func, const std::vector &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::vector &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const F &func, const std::shared_ptr &v) { + template + constexpr void encode(const Invocable auto &func, + const std::shared_ptr &v) { if (v == nullptr) { raise(::scale::EncodeError::DEREF_NULLPOINTER); } - encode(func, *v); + kagome::scale::encode(func, *v); } - template - constexpr void encode(const F &func, const std::unique_ptr &v) { + template + constexpr void encode(const Invocable auto &func, + const std::unique_ptr &v) { if (v == nullptr) { raise(::scale::EncodeError::DEREF_NULLPOINTER); } - encode(func, *v); + kagome::scale::encode(func, *v); } - template - constexpr void encode(const F &func, const std::list &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::list &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const F &func, const std::deque &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::deque &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const F &func, const std::tuple &v) { + template + constexpr void encode(const Invocable auto &func, + const std::tuple &v) { if constexpr (sizeof...(Ts) > 0) { - std::apply([&](const auto &...s) { (..., encode(func, s)); }, v); + std::apply( + [&](const auto &...s) { (..., kagome::scale::encode(func, s)); }, v); } } - template - void encode(const F &func, const std::optional &v) { + void encode(const Invocable auto &func, const std::optional &v) { enum class OptionalBool : uint8_t { NONE = 0u, OPT_TRUE = 1u, @@ -437,19 +465,29 @@ namespace kagome::scale { } else if (!*v) { result = OptionalBool::OPT_FALSE; } - encode(func, result); + kagome::scale::encode(func, result); } - template - void encode(const F &func, const std::optional &v) { + template + void encode(const Invocable auto &func, const std::optional &v) { if (!v.has_value()) { - encode(func, uint8_t(0u)); + kagome::scale::encode(func, uint8_t(0u)); } else { - encode(func, uint8_t(1u)); - encode(func, *v); + kagome::scale::encode(func, uint8_t(1u)); + kagome::scale::encode(func, *v); } } + void encode(const Invocable auto &func, const crypto::EcdsaSignature &data) { + kagome::scale::encode( + func, static_cast(data)); + } + + void encode(const Invocable auto &func, const crypto::EcdsaPublicKey &data) { + kagome::scale::encode( + func, static_cast(data)); + } + } // namespace kagome::scale #endif // KAGOME_SCALE_ENCODER_PRIMITIVES_HPP diff --git a/core/scale/kagome_scale.hpp b/core/scale/kagome_scale.hpp index fde89b7b95..aa1c90d5a7 100644 --- a/core/scale/kagome_scale.hpp +++ b/core/scale/kagome_scale.hpp @@ -12,7 +12,11 @@ #include "common/blob.hpp" #include "consensus/babe/types/babe_block_header.hpp" #include "consensus/babe/types/seal.hpp" +#include "consensus/beefy/types.hpp" +#include "consensus/grandpa/types/equivocation_proof.hpp" #include "network/types/blocks_response.hpp" +#include "network/types/collator_messages_vstaging.hpp" +#include "network/types/dispute_messages.hpp" #include "network/types/roles.hpp" #include "primitives/block_header.hpp" #include "primitives/block_id.hpp" @@ -20,6 +24,7 @@ #include "runtime/runtime_api/parachain_host_types.hpp" #include "scale/big_fixed_integers.hpp" #include "scale/encode_append.hpp" +#include "scale/encoder/concepts.hpp" #include "scale/libp2p_types.hpp" #include @@ -39,166 +44,255 @@ namespace kagome::scale { using ::scale::decode; - template - constexpr void encode(const F &func, const primitives::BlockHeader &bh); + constexpr void encode(const Invocable auto &func, + const primitives::BlockHeader &bh); - template - constexpr void encode(const F &func, const network::BlocksResponse &b); + constexpr void encode(const Invocable auto &func, + const consensus::grandpa::Equivocation &bh); + + constexpr void encode(const Invocable auto &func, + const primitives::BlockHeaderReflection &bh); + + constexpr void encode(const Invocable auto &func, + const primitives::BlockReflection &bh); + + constexpr void encode(const Invocable auto &func, + const network::BlocksResponse &b); - template + constexpr void encode(const Invocable auto &func, + const kagome::network::vstaging::CompactStatement &c); + + template constexpr void encode( - const F &func, + const Invocable auto &func, const common::SLVector &c); - template - constexpr void encode(const F &func, const Tagged &c); + template + constexpr void encode(const Invocable auto &func, + const Tagged &c); - template - constexpr void encode(const F &func, const common::SLBuffer &c); + template + constexpr void encode(const Invocable auto &func, + const common::SLBuffer &c); - template - constexpr void encode(const F &func, const network::Roles &c); + constexpr void encode(const Invocable auto &func, const network::Roles &c); - template - constexpr void encode(const F &func, const primitives::Other &c); + constexpr void encode(const Invocable auto &func, const primitives::Other &c); - template - constexpr void encode(const F &func, const primitives::Consensus &c); + constexpr void encode(const Invocable auto &func, + const primitives::Consensus &c); - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const runtime::PersistedValidationData &c); - template - constexpr void encode(const F &func, const primitives::Seal &c); + constexpr void encode(const Invocable auto &func, const primitives::Seal &c); - template - constexpr void encode(const F &func, const primitives::PreRuntime &c); + constexpr void encode(const Invocable auto &func, + const primitives::PreRuntime &c); - template - constexpr void encode(const F &func, const primitives::BlockInfo &c); + constexpr void encode(const Invocable auto &func, + const primitives::BlockInfo &c); - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const primitives::RuntimeEnvironmentUpdated &c); - template - constexpr void encode(const F &func, const ::scale::EncodeOpaqueValue &c); + constexpr void encode(const Invocable auto &func, + const ::scale::EncodeOpaqueValue &c); - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const consensus::babe::BabeBlockHeader &bh); + constexpr void encode(const Invocable auto &func, + const kagome::network::CandidateCommitments &c); + + constexpr void encode(const Invocable auto &func, + const kagome::network::CandidateReceipt &c); + + constexpr void encode(const Invocable auto &func, + const kagome::network::InvalidDisputeVote &c); + + constexpr void encode(const Invocable auto &func, + const kagome::network::ValidDisputeVote &c); + + constexpr void encode(const Invocable auto &func, + const consensus::grandpa::SignedPrecommit &c); template constexpr void encode(const F &func, const authority_discovery::AuthorityPeerInfo &c); + constexpr void encode(const Invocable auto &func, + const scale::PeerInfoSerializable &c); + + template + constexpr void encode(const Invocable auto &func, const Fixed &c); } // namespace kagome::scale #include "scale/encoder/primitives.hpp" namespace kagome::scale { - template - constexpr void encode(const F &func, const primitives::BlockHeader &bh) { - encode(func, bh.parent_hash); - encodeCompact(func, bh.number); - encode(func, bh.state_root); - encode(func, bh.extrinsics_root); - encode(func, bh.digest); + constexpr void encode(const Invocable auto &func, + const primitives::BlockHeader &bh) { + kagome::scale::encode(func, bh.parent_hash); + kagome::scale::encodeCompact(func, bh.number); + kagome::scale::encode(func, bh.state_root); + kagome::scale::encode(func, bh.extrinsics_root); + kagome::scale::encode(func, bh.digest); } - template - constexpr void encode(const F &func, const network::BlocksResponse &b) { - encode(func, b.blocks); + constexpr void encode(const Invocable auto &func, + const primitives::BlockReflection &bh) { + kagome::scale::encode(func, bh.header); + kagome::scale::encode(func, bh.body); } - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, + const primitives::BlockHeaderReflection &bh) { + kagome::scale::encode(func, bh.parent_hash); + kagome::scale::encodeCompact(func, bh.number); + kagome::scale::encode(func, bh.state_root); + kagome::scale::encode(func, bh.extrinsics_root); + kagome::scale::encode(func, bh.digest); + } + + constexpr void encode(const Invocable auto &func, + const network::BlocksResponse &b) { + kagome::scale::encode(func, b.blocks); + } + + constexpr void encode(const Invocable auto &func, const consensus::babe::BabeBlockHeader &bh) { - encode(func, bh.slot_assignment_type); - encode(func, bh.authority_index); - encode(func, bh.slot_number); + kagome::scale::encode(func, bh.slot_assignment_type); + kagome::scale::encode(func, bh.authority_index); + kagome::scale::encode(func, bh.slot_number); if (bh.needVRFCheck()) { - encode(func, bh.vrf_output); + kagome::scale::encode(func, bh.vrf_output); } } - template + template constexpr void encode( - const F &func, + const Invocable auto &func, const common::SLVector &c) { - encode(func, static_cast &>(c)); + kagome::scale::encode( + func, static_cast &>(c)); } - template - constexpr void encode(const F &func, const Tagged &c) { + template + constexpr void encode(const Invocable auto &func, + const Tagged &c) { if constexpr (std::is_scalar_v) { - encode(func, c.Wrapper::value); + kagome::scale::encode(func, c.Wrapper::value); } else { - encode(func, static_cast(c)); + kagome::scale::encode(func, static_cast(c)); } } - template - constexpr void encode(const F &func, const common::SLBuffer &c) { - encode(func, static_cast &>(c)); + template + constexpr void encode(const Invocable auto &func, + const common::SLBuffer &c) { + kagome::scale::encode( + func, static_cast &>(c)); } - template - constexpr void encode(const F &func, const primitives::Other &c) { - encode(func, static_cast(c)); + constexpr void encode(const Invocable auto &func, + const primitives::Other &c) { + kagome::scale::encode(func, static_cast(c)); } - template - constexpr void encode(const F &func, const primitives::Consensus &c) { - encode(func, static_cast(c)); + constexpr void encode(const Invocable auto &func, + const primitives::Consensus &c) { + kagome::scale::encode( + func, static_cast(c)); } - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const kagome::runtime::PersistedValidationData &c) { - encode(func, c.parent_head); - encode(func, c.relay_parent_number); - encode(func, c.relay_parent_storage_root); - encode(func, c.max_pov_size); + kagome::scale::encode(func, c.parent_head); + kagome::scale::encode(func, c.relay_parent_number); + kagome::scale::encode(func, c.relay_parent_storage_root); + kagome::scale::encode(func, c.max_pov_size); } - template - constexpr void encode(const F &func, const primitives::Seal &c) { - encode(func, static_cast(c)); + constexpr void encode(const Invocable auto &func, const primitives::Seal &c) { + kagome::scale::encode( + func, static_cast(c)); } - template - constexpr void encode(const F &func, const primitives::PreRuntime &c) { - encode(func, static_cast(c)); + constexpr void encode(const Invocable auto &func, + const primitives::PreRuntime &c) { + kagome::scale::encode( + func, static_cast(c)); } - template - constexpr void encode(const F &func, const primitives::BlockInfo &c) { - encode(func, c.number); - encode(func, c.hash); + constexpr void encode(const Invocable auto &func, + const primitives::BlockInfo &c) { + kagome::scale::encode(func, c.number); + kagome::scale::encode(func, c.hash); } - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const primitives::RuntimeEnvironmentUpdated &c) {} - template - constexpr void encode(const F &func, const ::scale::EncodeOpaqueValue &c) { + constexpr void encode(const Invocable auto &func, + const ::scale::EncodeOpaqueValue &c) { putByte(func, c.v.data(), c.v.size()); } - template - constexpr void encode(const F &func, const network::Roles &c) { - encode(func, c.value); + constexpr void encode(const Invocable auto &func, const network::Roles &c) { + kagome::scale::encode( + func, c.value); // NOLINT(cppcoreguidelines-pro-type-union-access) + } + + constexpr void encode(const Invocable auto &func, + const consensus::grandpa::Equivocation &bh) { + kagome::scale::encode(func, bh.stage); + kagome::scale::encode(func, bh.round_number); + kagome::scale::encode(func, bh.first); + kagome::scale::encode(func, bh.second); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::CandidateCommitments &c) { + kagome::scale::encode(func, c.upward_msgs); + kagome::scale::encode(func, c.outbound_hor_msgs); + kagome::scale::encode(func, c.opt_para_runtime); + kagome::scale::encode(func, c.para_head); + kagome::scale::encode(func, c.downward_msgs_count); + kagome::scale::encode(func, c.watermark); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::CandidateReceipt &c) { + kagome::scale::encode(func, c.descriptor); + kagome::scale::encode(func, c.commitments_hash); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::vstaging::CompactStatement &c) { + kagome::scale::encode(func, c.header); + kagome::scale::encode(func, c.inner_value); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::InvalidDisputeVote &c) { + kagome::scale::encode(func, c.index); + kagome::scale::encode(func, c.signature); + kagome::scale::encode(func, c.kind); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::ValidDisputeVote &c) { + kagome::scale::encode(func, c.index); + kagome::scale::encode(func, c.signature); + kagome::scale::encode(func, c.kind); + } + + constexpr void encode(const Invocable auto &func, + const consensus::grandpa::SignedPrecommit &c) { + kagome::scale::encode( + func, static_cast(c)); } template @@ -209,6 +303,25 @@ namespace kagome::scale { encode(func, c.peer); } + constexpr void encode(const Invocable auto &func, + const scale::PeerInfoSerializable &c) { + std::vector addresses; + addresses.reserve(c.addresses.size()); + for (const auto &addr : c.addresses) { + addresses.emplace_back(addr.getStringAddress()); + } + encode(func, c.id.toBase58()); + encode(func, addresses); + } + + template + constexpr void encode(const Invocable auto &func, const Fixed &c) { + constexpr size_t bits = Fixed::kByteSize * 8; + for (size_t i = 0; i < bits; i += 8) { + encode(func, ::scale::convert_to((*c >> i) & 0xFFu)); + } + } + } // namespace kagome::scale #endif // KAGOME_KAGOME_SCALE_HPP diff --git a/core/utils/struct_to_tuple.hpp b/core/utils/struct_to_tuple.hpp index 0f43ba7929..5f0554bb4f 100644 --- a/core/utils/struct_to_tuple.hpp +++ b/core/utils/struct_to_tuple.hpp @@ -59,10 +59,8 @@ return std::make_tuple(REPEATY_REF(ONES, p)); \ } -#define TO_TUPLE1 \ - TO_TUPLE_N(1) else { \ - return std::make_tuple(); \ - } +#define TO_TUPLE1 \ + TO_TUPLE_N(1) else { return std::make_tuple(); } #define TO_TUPLE2 TO_TUPLE_N(2) else TO_TUPLE1 #define TO_TUPLE3 TO_TUPLE_N(3) else TO_TUPLE2 #define TO_TUPLE4 TO_TUPLE_N(4) else TO_TUPLE3 @@ -75,7 +73,7 @@ namespace kagome::utils { template - decltype(void(T{{std::declval()}...}), std::true_type{}) + decltype(void(T{std::declval()...}), std::true_type()) test_is_braces_constructible(int); template diff --git a/test/core/runtime/wasm_test.cpp b/test/core/runtime/wasm_test.cpp index 1bc5bbc524..6bbffefdf4 100644 --- a/test/core/runtime/wasm_test.cpp +++ b/test/core/runtime/wasm_test.cpp @@ -1,4 +1,72 @@ -#include +#include +#include + +/// #include Conflicting with fmt::ranges::formatter +namespace qtils { + struct Hex { + qtils::BytesIn v; + }; +} // namespace qtils + +template <> +struct fmt::formatter { + bool prefix = true; + bool full = false; + bool lower = true; + + constexpr auto parse(format_parse_context &ctx) { + auto it = ctx.begin(); + auto end = [&] { return it == ctx.end() or *it == '}'; }; + if (end()) { + return it; + } + prefix = *it == '0'; + if (prefix) { + ++it; + } + if (not end()) { + lower = *it == 'x'; + if (lower or *it == 'X') { + ++it; + full = true; + if (end()) { + return it; + } + } + } + fmt::throw_format_error("\"x\"/\"X\" or \"0x\"/\"0X\" expected"); + } + + auto format(const qtils::BytesIn &bytes, format_context &ctx) const { + auto out = ctx.out(); + if (prefix) { + out = fmt::detail::write(out, "0x"); + } + constexpr size_t kHead = 2, kTail = 2, kSmall = 1; + if (full or bytes.size() <= kHead + kTail + kSmall) { + return lower ? fmt::format_to(out, "{:02x}", fmt::join(bytes, "")) + : fmt::format_to(out, "{:02X}", fmt::join(bytes, "")); + } + return fmt::format_to(out, + "{:02x}…{:02x}", + fmt::join(bytes.first(kHead), ""), + fmt::join(bytes.last(kTail), "")); + } +}; + +template <> +struct fmt::formatter : fmt::formatter {}; + +template <> +struct fmt::formatter : fmt::formatter {}; + +template <> +struct fmt::formatter : fmt::formatter { + auto format(const qtils::Hex &v, format_context &ctx) const { + return formatter::format(v.v, ctx); + } +}; + #include #include "common/bytestr.hpp" #include "crypto/hasher/hasher_impl.hpp" diff --git a/test/testutil/scale_test_comparator.hpp b/test/testutil/scale_test_comparator.hpp index ad2f4cf578..fede908b0b 100644 --- a/test/testutil/scale_test_comparator.hpp +++ b/test/testutil/scale_test_comparator.hpp @@ -14,7 +14,7 @@ namespace testutil { template inline outcome::result> scaleEncodeAndCompareWithRef( T &&...t) { - return ::scale::encode(std::forward(t)...); + return scale::encode(std::forward(t)...); } } // namespace testutil