From 7fe61ddf4281872586b9930a524fb91417884917 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 3 Oct 2023 12:32:11 +0200 Subject: [PATCH 01/22] chainHead/storage: Fix typo Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/chain_head/chain_head_storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index 6e19f59a5d68..1e08bd8cd665 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -37,7 +37,7 @@ use super::{ FollowEvent, }; -/// The query type of an interation. +/// The query type of an iteration. enum IterQueryType { /// Iterating over (key, value) pairs. Value, From e8d3d8a4e39eaf416727463f57e31ccbfd3cb385 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 3 Oct 2023 15:07:56 +0200 Subject: [PATCH 02/22] rpc-v2: Move hex_string to common Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/archive/archive.rs | 3 +-- .../rpc-spec-v2/src/chain_head/chain_head.rs | 3 +-- .../src/chain_head/chain_head_storage.rs | 16 ++++++++-------- .../client/rpc-spec-v2/src/chain_head/mod.rs | 7 ------- substrate/client/rpc-spec-v2/src/lib.rs | 6 ++++++ 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs index bded842d8fd0..5e3f640c7257 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -20,8 +20,7 @@ use crate::{ archive::{error::Error as ArchiveError, ArchiveApiServer}, - chain_head::hex_string, - MethodResult, + hex_string, MethodResult, }; use codec::Encode; diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs index a8c1c4f7e083..7f65e2fed10d 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs @@ -28,10 +28,9 @@ use crate::{ chain_head_follow::ChainHeadFollower, error::Error as ChainHeadRpcError, event::{FollowEvent, MethodResponse, OperationError, StorageQuery}, - hex_string, subscription::{SubscriptionManagement, SubscriptionManagementError}, }, - SubscriptionTaskExecutor, + hex_string, SubscriptionTaskExecutor, }; use codec::Encode; use futures::future::FutureExt; diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index 1e08bd8cd665..a7b92df55fc6 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -25,16 +25,16 @@ use sc_utils::mpsc::TracingUnboundedSender; use sp_api::BlockT; use sp_core::storage::well_known_keys; -use crate::chain_head::event::OperationStorageItems; - -use super::{ - event::{ - OperationError, OperationId, StorageQuery, StorageQueryType, StorageResult, - StorageResultType, +use crate::{ + chain_head::{ + event::{ + OperationError, OperationId, OperationStorageItems, StorageQuery, StorageQueryType, + StorageResult, StorageResultType, + }, + subscription::BlockGuard, + FollowEvent, }, hex_string, - subscription::BlockGuard, - FollowEvent, }; /// The query type of an iteration. diff --git a/substrate/client/rpc-spec-v2/src/chain_head/mod.rs b/substrate/client/rpc-spec-v2/src/chain_head/mod.rs index 1bd228857802..4cbbd00f64f3 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/mod.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/mod.rs @@ -42,10 +42,3 @@ pub use event::{ BestBlockChanged, ErrorEvent, Finalized, FollowEvent, Initialized, NewBlock, RuntimeEvent, RuntimeVersionEvent, }; - -use sp_core::hexdisplay::{AsBytesRef, HexDisplay}; - -/// Util function to print the results of `chianHead` as hex string -pub(crate) fn hex_string(data: &Data) -> String { - format!("0x{:?}", HexDisplay::from(data)) -} diff --git a/substrate/client/rpc-spec-v2/src/lib.rs b/substrate/client/rpc-spec-v2/src/lib.rs index d202bfef4a74..1d98697b4f4b 100644 --- a/substrate/client/rpc-spec-v2/src/lib.rs +++ b/substrate/client/rpc-spec-v2/src/lib.rs @@ -24,6 +24,7 @@ #![deny(unused_crate_dependencies)] use serde::{Deserialize, Serialize}; +use sp_core::hexdisplay::{AsBytesRef, HexDisplay}; pub mod archive; pub mod chain_head; @@ -75,6 +76,11 @@ pub struct MethodResultErr { pub error: String, } +/// Util function to print the results of rpc-spec-v2 as hex string +pub fn hex_string(data: &Data) -> String { + format!("0x{:?}", HexDisplay::from(data)) +} + #[cfg(test)] mod tests { use super::*; From 56e03db9083bc1a964dbdbcdfdafe371cdc30411 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 3 Oct 2023 15:31:56 +0200 Subject: [PATCH 03/22] rpc-v2: Introduce common module for storage queries and events Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/chain_head/api.rs | 5 +- .../rpc-spec-v2/src/chain_head/chain_head.rs | 3 +- .../src/chain_head/chain_head_storage.rs | 6 +- .../rpc-spec-v2/src/chain_head/event.rs | 144 +----------- .../client/rpc-spec-v2/src/common/events.rs | 168 ++++++++++++++ .../client/rpc-spec-v2/src/common/mod.rs | 17 ++ .../client/rpc-spec-v2/src/common/storage.rs | 209 ++++++++++++++++++ substrate/client/rpc-spec-v2/src/lib.rs | 2 + 8 files changed, 406 insertions(+), 148 deletions(-) create mode 100644 substrate/client/rpc-spec-v2/src/common/events.rs create mode 100644 substrate/client/rpc-spec-v2/src/common/mod.rs create mode 100644 substrate/client/rpc-spec-v2/src/common/storage.rs diff --git a/substrate/client/rpc-spec-v2/src/chain_head/api.rs b/substrate/client/rpc-spec-v2/src/chain_head/api.rs index d93c4018b60f..982a88d654e5 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/api.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/api.rs @@ -19,7 +19,10 @@ #![allow(non_snake_case)] //! API trait of the chain head. -use crate::chain_head::event::{FollowEvent, MethodResponse, StorageQuery}; +use crate::{ + chain_head::event::{FollowEvent, MethodResponse}, + common::events::StorageQuery, +}; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; #[rpc(client, server)] diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs index 7f65e2fed10d..c8462e954ac1 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs @@ -27,9 +27,10 @@ use crate::{ api::ChainHeadApiServer, chain_head_follow::ChainHeadFollower, error::Error as ChainHeadRpcError, - event::{FollowEvent, MethodResponse, OperationError, StorageQuery}, + event::{FollowEvent, MethodResponse, OperationError}, subscription::{SubscriptionManagement, SubscriptionManagementError}, }, + common::events::StorageQuery, hex_string, SubscriptionTaskExecutor, }; use codec::Encode; diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index a7b92df55fc6..b7ec11f81d76 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -27,13 +27,11 @@ use sp_core::storage::well_known_keys; use crate::{ chain_head::{ - event::{ - OperationError, OperationId, OperationStorageItems, StorageQuery, StorageQueryType, - StorageResult, StorageResultType, - }, + event::{OperationError, OperationId, OperationStorageItems}, subscription::BlockGuard, FollowEvent, }, + common::events::{StorageQuery, StorageQueryType, StorageResult, StorageResultType}, hex_string, }; diff --git a/substrate/client/rpc-spec-v2/src/chain_head/event.rs b/substrate/client/rpc-spec-v2/src/chain_head/event.rs index b5f9d6cc2fff..2de06bacb52a 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/event.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/event.rs @@ -23,6 +23,8 @@ use sp_api::ApiError; use sp_version::RuntimeVersion; use std::collections::BTreeMap; +use crate::common::events::StorageResult; + /// The operation could not be processed due to an error. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -313,56 +315,6 @@ pub enum FollowEvent { Stop, } -/// The storage item received as paramter. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StorageQuery { - /// The provided key. - pub key: Key, - /// The type of the storage query. - #[serde(rename = "type")] - pub query_type: StorageQueryType, -} - -/// The type of the storage query. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum StorageQueryType { - /// Fetch the value of the provided key. - Value, - /// Fetch the hash of the value of the provided key. - Hash, - /// Fetch the closest descendant merkle value. - ClosestDescendantMerkleValue, - /// Fetch the values of all descendants of they provided key. - DescendantsValues, - /// Fetch the hashes of the values of all descendants of they provided key. - DescendantsHashes, -} - -/// The storage result. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StorageResult { - /// The hex-encoded key of the result. - pub key: String, - /// The result of the query. - #[serde(flatten)] - pub result: StorageResultType, -} - -/// The type of the storage query. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum StorageResultType { - /// Fetch the value of the provided key. - Value(String), - /// Fetch the hash of the value of the provided key. - Hash(String), - /// Fetch the closest descendant merkle value. - ClosestDescendantMerkleValue(String), -} - /// The method respose of `chainHead_body`, `chainHead_call` and `chainHead_storage`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -697,96 +649,4 @@ mod tests { let event_dec: MethodResponse = serde_json::from_str(exp).unwrap(); assert_eq!(event_dec, event); } - - #[test] - fn chain_head_storage_query() { - // Item with Value. - let item = StorageQuery { key: "0x1", query_type: StorageQueryType::Value }; - // Encode - let ser = serde_json::to_string(&item).unwrap(); - let exp = r#"{"key":"0x1","type":"value"}"#; - assert_eq!(ser, exp); - // Decode - let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); - assert_eq!(dec, item); - - // Item with Hash. - let item = StorageQuery { key: "0x1", query_type: StorageQueryType::Hash }; - // Encode - let ser = serde_json::to_string(&item).unwrap(); - let exp = r#"{"key":"0x1","type":"hash"}"#; - assert_eq!(ser, exp); - // Decode - let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); - assert_eq!(dec, item); - - // Item with DescendantsValues. - let item = StorageQuery { key: "0x1", query_type: StorageQueryType::DescendantsValues }; - // Encode - let ser = serde_json::to_string(&item).unwrap(); - let exp = r#"{"key":"0x1","type":"descendantsValues"}"#; - assert_eq!(ser, exp); - // Decode - let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); - assert_eq!(dec, item); - - // Item with DescendantsHashes. - let item = StorageQuery { key: "0x1", query_type: StorageQueryType::DescendantsHashes }; - // Encode - let ser = serde_json::to_string(&item).unwrap(); - let exp = r#"{"key":"0x1","type":"descendantsHashes"}"#; - assert_eq!(ser, exp); - // Decode - let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); - assert_eq!(dec, item); - - // Item with Merkle. - let item = - StorageQuery { key: "0x1", query_type: StorageQueryType::ClosestDescendantMerkleValue }; - // Encode - let ser = serde_json::to_string(&item).unwrap(); - let exp = r#"{"key":"0x1","type":"closestDescendantMerkleValue"}"#; - assert_eq!(ser, exp); - // Decode - let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); - assert_eq!(dec, item); - } - - #[test] - fn chain_head_storage_result() { - // Item with Value. - let item = - StorageResult { key: "0x1".into(), result: StorageResultType::Value("res".into()) }; - // Encode - let ser = serde_json::to_string(&item).unwrap(); - let exp = r#"{"key":"0x1","value":"res"}"#; - assert_eq!(ser, exp); - // Decode - let dec: StorageResult = serde_json::from_str(exp).unwrap(); - assert_eq!(dec, item); - - // Item with Hash. - let item = - StorageResult { key: "0x1".into(), result: StorageResultType::Hash("res".into()) }; - // Encode - let ser = serde_json::to_string(&item).unwrap(); - let exp = r#"{"key":"0x1","hash":"res"}"#; - assert_eq!(ser, exp); - // Decode - let dec: StorageResult = serde_json::from_str(exp).unwrap(); - assert_eq!(dec, item); - - // Item with DescendantsValues. - let item = StorageResult { - key: "0x1".into(), - result: StorageResultType::ClosestDescendantMerkleValue("res".into()), - }; - // Encode - let ser = serde_json::to_string(&item).unwrap(); - let exp = r#"{"key":"0x1","closestDescendantMerkleValue":"res"}"#; - assert_eq!(ser, exp); - // Decode - let dec: StorageResult = serde_json::from_str(exp).unwrap(); - assert_eq!(dec, item); - } } diff --git a/substrate/client/rpc-spec-v2/src/common/events.rs b/substrate/client/rpc-spec-v2/src/common/events.rs new file mode 100644 index 000000000000..e2f367ed02b7 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/common/events.rs @@ -0,0 +1,168 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Common events for RPC-V2 spec. + +use serde::{Deserialize, Serialize}; + +/// The storage item received as paramter. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageQuery { + /// The provided key. + pub key: Key, + /// The type of the storage query. + #[serde(rename = "type")] + pub query_type: StorageQueryType, +} + +/// The type of the storage query. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum StorageQueryType { + /// Fetch the value of the provided key. + Value, + /// Fetch the hash of the value of the provided key. + Hash, + /// Fetch the closest descendant merkle value. + ClosestDescendantMerkleValue, + /// Fetch the values of all descendants of they provided key. + DescendantsValues, + /// Fetch the hashes of the values of all descendants of they provided key. + DescendantsHashes, +} + +/// The storage result. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageResult { + /// The hex-encoded key of the result. + pub key: String, + /// The result of the query. + #[serde(flatten)] + pub result: StorageResultType, +} + +/// The type of the storage query. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum StorageResultType { + /// Fetch the value of the provided key. + Value(String), + /// Fetch the hash of the value of the provided key. + Hash(String), + /// Fetch the closest descendant merkle value. + ClosestDescendantMerkleValue(String), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn storage_result() { + // Item with Value. + let item = + StorageResult { key: "0x1".into(), result: StorageResultType::Value("res".into()) }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","value":"res"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageResult = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + + // Item with Hash. + let item = + StorageResult { key: "0x1".into(), result: StorageResultType::Hash("res".into()) }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","hash":"res"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageResult = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + + // Item with DescendantsValues. + let item = StorageResult { + key: "0x1".into(), + result: StorageResultType::ClosestDescendantMerkleValue("res".into()), + }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","closestDescendantMerkleValue":"res"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageResult = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + } + + #[test] + fn storage_query() { + // Item with Value. + let item = StorageQuery { key: "0x1", query_type: StorageQueryType::Value }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","type":"value"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + + // Item with Hash. + let item = StorageQuery { key: "0x1", query_type: StorageQueryType::Hash }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","type":"hash"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + + // Item with DescendantsValues. + let item = StorageQuery { key: "0x1", query_type: StorageQueryType::DescendantsValues }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","type":"descendantsValues"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + + // Item with DescendantsHashes. + let item = StorageQuery { key: "0x1", query_type: StorageQueryType::DescendantsHashes }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","type":"descendantsHashes"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + + // Item with Merkle. + let item = + StorageQuery { key: "0x1", query_type: StorageQueryType::ClosestDescendantMerkleValue }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","type":"closestDescendantMerkleValue"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + } +} diff --git a/substrate/client/rpc-spec-v2/src/common/mod.rs b/substrate/client/rpc-spec-v2/src/common/mod.rs new file mode 100644 index 000000000000..ac1af8fce3c9 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/common/mod.rs @@ -0,0 +1,17 @@ +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Common types and functionality for the RPC-V2 spec. + +pub mod events; +pub mod storage; diff --git a/substrate/client/rpc-spec-v2/src/common/storage.rs b/substrate/client/rpc-spec-v2/src/common/storage.rs new file mode 100644 index 000000000000..ad13d1a124fc --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/common/storage.rs @@ -0,0 +1,209 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Storage queries for the RPC-V2 spec. + +use std::{marker::PhantomData, sync::Arc}; + +use sc_client_api::{Backend, ChildInfo, StorageKey, StorageProvider}; +use sp_api::BlockT; +use sp_core::storage::well_known_keys; + +use super::events::{StorageResult, StorageResultType}; +use crate::hex_string; + +/// Call into the storage of blocks. +pub struct Storage { + /// Substrate client. + client: Arc, + _phandom: PhantomData<(BE, Block)>, +} + +impl Storage { + /// Constructs a new [`Storage`]. + pub fn new(client: Arc) -> Self { + Self { client, _phandom: PhantomData } + } +} + +/// Query to iterate over storage. +pub struct QueryIter { + /// The key from which the iteration was started. + query_key: StorageKey, + /// The key after which pagination should resume. + pagination_start_key: Option, + /// The type of the query (either value or hash). + ty: IterQueryType, +} + +/// The query type of an iteration. +pub enum IterQueryType { + /// Iterating over (key, value) pairs. + Value, + /// Iterating over (key, hash) pairs. + Hash, +} + +/// Checks if the provided key (main or child key) is valid +/// for queries. +/// +/// Keys that are identical to `:child_storage:` or `:child_storage:default:` +/// are not queryable. +fn is_key_queryable(key: &[u8]) -> bool { + !well_known_keys::is_default_child_storage_key(key) && + !well_known_keys::is_child_storage_key(key) +} + +/// The result of making a query call. +pub type QueryResult = Result, String>; + +/// The result of iterating over keys. +pub type QueryIterResult = Result<(Vec, Option), String>; + +impl Storage +where + Block: BlockT + 'static, + BE: Backend + 'static, + Client: StorageProvider + 'static, +{ + /// Fetch the value from storage. + pub fn query_value( + &self, + hash: Block::Hash, + key: &StorageKey, + child_key: Option<&ChildInfo>, + ) -> QueryResult { + let result = if let Some(child_key) = child_key { + self.client.child_storage(hash, child_key, key) + } else { + self.client.storage(hash, key) + }; + + result + .map(|opt| { + QueryResult::Ok(opt.map(|storage_data| StorageResult { + key: hex_string(&key.0), + result: StorageResultType::Value(hex_string(&storage_data.0)), + })) + }) + .unwrap_or_else(|error| QueryResult::Err(error.to_string())) + } + + /// Fetch the hash of a value from storage. + pub fn query_hash( + &self, + hash: Block::Hash, + key: &StorageKey, + child_key: Option<&ChildInfo>, + ) -> QueryResult { + let result = if let Some(child_key) = child_key { + self.client.child_storage_hash(hash, child_key, key) + } else { + self.client.storage_hash(hash, key) + }; + + result + .map(|opt| { + QueryResult::Ok(opt.map(|storage_data| StorageResult { + key: hex_string(&key.0), + result: StorageResultType::Hash(hex_string(&storage_data.as_ref())), + })) + }) + .unwrap_or_else(|error| QueryResult::Err(error.to_string())) + } + + /// Fetch the closest merkle value. + pub fn query_merkle_value( + &self, + hash: Block::Hash, + key: &StorageKey, + child_key: Option<&ChildInfo>, + ) -> QueryResult { + let result = if let Some(child_key) = child_key { + self.client.child_closest_merkle_value(hash, child_key, key) + } else { + self.client.closest_merkle_value(hash, key) + }; + + result + .map(|opt| { + QueryResult::Ok(opt.map(|storage_data| { + let result = match &storage_data { + sc_client_api::MerkleValue::Node(data) => hex_string(&data.as_slice()), + sc_client_api::MerkleValue::Hash(hash) => hex_string(&hash.as_ref()), + }; + + StorageResult { + key: hex_string(&key.0), + result: StorageResultType::ClosestDescendantMerkleValue(result), + } + })) + }) + .unwrap_or_else(|error| QueryResult::Err(error.to_string())) + } + + /// Iterate over at most the provided number of keys. + /// + /// Returns the storage result with a potential next key to resume iteration. + pub fn query_iter_pagination( + &self, + query: QueryIter, + hash: Block::Hash, + child_key: Option<&ChildInfo>, + operation_max_storage_items: usize, + ) -> QueryIterResult { + let QueryIter { ty, query_key, pagination_start_key } = query; + + let mut keys_iter = if let Some(child_key) = child_key { + self.client.child_storage_keys( + hash, + child_key.to_owned(), + Some(&query_key), + pagination_start_key.as_ref(), + ) + } else { + self.client.storage_keys(hash, Some(&query_key), pagination_start_key.as_ref()) + } + .map_err(|err| err.to_string())?; + + let mut ret = Vec::with_capacity(operation_max_storage_items); + let mut next_pagination_key = None; + for _ in 0..operation_max_storage_items { + let Some(key) = keys_iter.next() else { break }; + + next_pagination_key = Some(key.clone()); + + let result = match ty { + IterQueryType::Value => self.query_value(hash, &key, child_key), + IterQueryType::Hash => self.query_hash(hash, &key, child_key), + }?; + + if let Some(value) = result { + ret.push(value); + } + } + + // Save the next key if any to continue the iteration. + let maybe_next_query = keys_iter.next().map(|_| QueryIter { + ty, + query_key, + pagination_start_key: next_pagination_key, + }); + Ok((ret, maybe_next_query)) + } +} diff --git a/substrate/client/rpc-spec-v2/src/lib.rs b/substrate/client/rpc-spec-v2/src/lib.rs index 1d98697b4f4b..9b851443a2d9 100644 --- a/substrate/client/rpc-spec-v2/src/lib.rs +++ b/substrate/client/rpc-spec-v2/src/lib.rs @@ -26,6 +26,8 @@ use serde::{Deserialize, Serialize}; use sp_core::hexdisplay::{AsBytesRef, HexDisplay}; +mod common; + pub mod archive; pub mod chain_head; pub mod chain_spec; From 316f116a25e51366fa2d9535c7177f1dbc23c3ea Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 3 Oct 2023 15:38:25 +0200 Subject: [PATCH 04/22] chainHead/storage: Use common storage object for queries Signed-off-by: Alexandru Vasile --- .../src/chain_head/chain_head_storage.rs | 185 ++---------------- .../client/rpc-spec-v2/src/common/storage.rs | 8 +- 2 files changed, 20 insertions(+), 173 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index b7ec11f81d76..dfd10625cab8 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -23,7 +23,6 @@ use std::{collections::VecDeque, marker::PhantomData, sync::Arc}; use sc_client_api::{Backend, ChildInfo, StorageKey, StorageProvider}; use sc_utils::mpsc::TracingUnboundedSender; use sp_api::BlockT; -use sp_core::storage::well_known_keys; use crate::{ chain_head::{ @@ -31,22 +30,16 @@ use crate::{ subscription::BlockGuard, FollowEvent, }, - common::events::{StorageQuery, StorageQueryType, StorageResult, StorageResultType}, - hex_string, + common::{ + events::{StorageQuery, StorageQueryType}, + storage::{is_key_queryable, IterQueryType, QueryIter, QueryIterResult, Storage}, + }, }; -/// The query type of an iteration. -enum IterQueryType { - /// Iterating over (key, value) pairs. - Value, - /// Iterating over (key, hash) pairs. - Hash, -} - /// Generates the events of the `chainHead_storage` method. pub struct ChainHeadStorage { - /// Substrate client. - client: Arc, + /// Storage client. + client: Storage, /// Queue of operations that may require pagination. iter_operations: VecDeque, /// The maximum number of items reported by the `chainHead_storage` before @@ -59,7 +52,7 @@ impl ChainHeadStorage { /// Constructs a new [`ChainHeadStorage`]. pub fn new(client: Arc, operation_max_storage_items: usize) -> Self { Self { - client, + client: Storage::new(client), iter_operations: VecDeque::new(), operation_max_storage_items, _phandom: PhantomData, @@ -67,163 +60,12 @@ impl ChainHeadStorage { } } -/// Query to iterate over storage. -struct QueryIter { - /// The key from which the iteration was started. - query_key: StorageKey, - /// The key after which pagination should resume. - pagination_start_key: Option, - /// The type of the query (either value or hash). - ty: IterQueryType, -} - -/// Checks if the provided key (main or child key) is valid -/// for queries. -/// -/// Keys that are identical to `:child_storage:` or `:child_storage:default:` -/// are not queryable. -fn is_key_queryable(key: &[u8]) -> bool { - !well_known_keys::is_default_child_storage_key(key) && - !well_known_keys::is_child_storage_key(key) -} - -/// The result of making a query call. -type QueryResult = Result, String>; - -/// The result of iterating over keys. -type QueryIterResult = Result<(Vec, Option), String>; - impl ChainHeadStorage where Block: BlockT + 'static, BE: Backend + 'static, Client: StorageProvider + 'static, { - /// Fetch the value from storage. - fn query_storage_value( - &self, - hash: Block::Hash, - key: &StorageKey, - child_key: Option<&ChildInfo>, - ) -> QueryResult { - let result = if let Some(child_key) = child_key { - self.client.child_storage(hash, child_key, key) - } else { - self.client.storage(hash, key) - }; - - result - .map(|opt| { - QueryResult::Ok(opt.map(|storage_data| StorageResult { - key: hex_string(&key.0), - result: StorageResultType::Value(hex_string(&storage_data.0)), - })) - }) - .unwrap_or_else(|error| QueryResult::Err(error.to_string())) - } - - /// Fetch the hash of a value from storage. - fn query_storage_hash( - &self, - hash: Block::Hash, - key: &StorageKey, - child_key: Option<&ChildInfo>, - ) -> QueryResult { - let result = if let Some(child_key) = child_key { - self.client.child_storage_hash(hash, child_key, key) - } else { - self.client.storage_hash(hash, key) - }; - - result - .map(|opt| { - QueryResult::Ok(opt.map(|storage_data| StorageResult { - key: hex_string(&key.0), - result: StorageResultType::Hash(hex_string(&storage_data.as_ref())), - })) - }) - .unwrap_or_else(|error| QueryResult::Err(error.to_string())) - } - - /// Fetch the closest merkle value. - fn query_storage_merkle_value( - &self, - hash: Block::Hash, - key: &StorageKey, - child_key: Option<&ChildInfo>, - ) -> QueryResult { - let result = if let Some(child_key) = child_key { - self.client.child_closest_merkle_value(hash, child_key, key) - } else { - self.client.closest_merkle_value(hash, key) - }; - - result - .map(|opt| { - QueryResult::Ok(opt.map(|storage_data| { - let result = match &storage_data { - sc_client_api::MerkleValue::Node(data) => hex_string(&data.as_slice()), - sc_client_api::MerkleValue::Hash(hash) => hex_string(&hash.as_ref()), - }; - - StorageResult { - key: hex_string(&key.0), - result: StorageResultType::ClosestDescendantMerkleValue(result), - } - })) - }) - .unwrap_or_else(|error| QueryResult::Err(error.to_string())) - } - - /// Iterate over at most `operation_max_storage_items` keys. - /// - /// Returns the storage result with a potential next key to resume iteration. - fn query_storage_iter_pagination( - &self, - query: QueryIter, - hash: Block::Hash, - child_key: Option<&ChildInfo>, - ) -> QueryIterResult { - let QueryIter { ty, query_key, pagination_start_key } = query; - - let mut keys_iter = if let Some(child_key) = child_key { - self.client.child_storage_keys( - hash, - child_key.to_owned(), - Some(&query_key), - pagination_start_key.as_ref(), - ) - } else { - self.client.storage_keys(hash, Some(&query_key), pagination_start_key.as_ref()) - } - .map_err(|err| err.to_string())?; - - let mut ret = Vec::with_capacity(self.operation_max_storage_items); - let mut next_pagination_key = None; - for _ in 0..self.operation_max_storage_items { - let Some(key) = keys_iter.next() else { break }; - - next_pagination_key = Some(key.clone()); - - let result = match ty { - IterQueryType::Value => self.query_storage_value(hash, &key, child_key), - IterQueryType::Hash => self.query_storage_hash(hash, &key, child_key), - }?; - - if let Some(value) = result { - ret.push(value); - } - } - - // Save the next key if any to continue the iteration. - let maybe_next_query = keys_iter.next().map(|_| QueryIter { - ty, - query_key, - pagination_start_key: next_pagination_key, - }); - Ok((ret, maybe_next_query)) - } - /// Iterate over (key, hash) and (key, value) generating the `WaitingForContinue` event if /// necessary. async fn generate_storage_iter_events( @@ -240,7 +82,12 @@ where return } - let result = self.query_storage_iter_pagination(query, hash, child_key.as_ref()); + let result = self.client.query_iter_pagination( + query, + hash, + child_key.as_ref(), + self.operation_max_storage_items, + ); let (events, maybe_next_query) = match result { QueryIterResult::Ok(result) => result, QueryIterResult::Err(error) => { @@ -309,7 +156,7 @@ where match item.query_type { StorageQueryType::Value => { - match self.query_storage_value(hash, &item.key, child_key.as_ref()) { + match self.client.query_value(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, Err(error) => { @@ -319,7 +166,7 @@ where } }, StorageQueryType::Hash => - match self.query_storage_hash(hash, &item.key, child_key.as_ref()) { + match self.client.query_hash(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, Err(error) => { @@ -328,7 +175,7 @@ where }, }, StorageQueryType::ClosestDescendantMerkleValue => - match self.query_storage_merkle_value(hash, &item.key, child_key.as_ref()) { + match self.client.query_merkle_value(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, Err(error) => { diff --git a/substrate/client/rpc-spec-v2/src/common/storage.rs b/substrate/client/rpc-spec-v2/src/common/storage.rs index ad13d1a124fc..6475fa7f173d 100644 --- a/substrate/client/rpc-spec-v2/src/common/storage.rs +++ b/substrate/client/rpc-spec-v2/src/common/storage.rs @@ -44,11 +44,11 @@ impl Storage { /// Query to iterate over storage. pub struct QueryIter { /// The key from which the iteration was started. - query_key: StorageKey, + pub query_key: StorageKey, /// The key after which pagination should resume. - pagination_start_key: Option, + pub pagination_start_key: Option, /// The type of the query (either value or hash). - ty: IterQueryType, + pub ty: IterQueryType, } /// The query type of an iteration. @@ -64,7 +64,7 @@ pub enum IterQueryType { /// /// Keys that are identical to `:child_storage:` or `:child_storage:default:` /// are not queryable. -fn is_key_queryable(key: &[u8]) -> bool { +pub fn is_key_queryable(key: &[u8]) -> bool { !well_known_keys::is_default_child_storage_key(key) && !well_known_keys::is_child_storage_key(key) } From c5886c0e1f1df764c304db53f7871852f6a68710 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 3 Oct 2023 15:44:42 +0200 Subject: [PATCH 05/22] rpc-v2: Rename storage params for pagination Signed-off-by: Alexandru Vasile --- substrate/client/rpc-spec-v2/src/common/storage.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/common/storage.rs b/substrate/client/rpc-spec-v2/src/common/storage.rs index 6475fa7f173d..e2412beb4103 100644 --- a/substrate/client/rpc-spec-v2/src/common/storage.rs +++ b/substrate/client/rpc-spec-v2/src/common/storage.rs @@ -165,7 +165,7 @@ where query: QueryIter, hash: Block::Hash, child_key: Option<&ChildInfo>, - operation_max_storage_items: usize, + count: usize, ) -> QueryIterResult { let QueryIter { ty, query_key, pagination_start_key } = query; @@ -181,9 +181,9 @@ where } .map_err(|err| err.to_string())?; - let mut ret = Vec::with_capacity(operation_max_storage_items); + let mut ret = Vec::with_capacity(count); let mut next_pagination_key = None; - for _ in 0..operation_max_storage_items { + for _ in 0..count { let Some(key) = keys_iter.next() else { break }; next_pagination_key = Some(key.clone()); From 077f2f5f3795e41dc988974bd647eb4463098933 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 3 Oct 2023 16:01:28 +0200 Subject: [PATCH 06/22] common/event: Add paginated query type Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/common/events.rs | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/substrate/client/rpc-spec-v2/src/common/events.rs b/substrate/client/rpc-spec-v2/src/common/events.rs index e2f367ed02b7..fa208427dccb 100644 --- a/substrate/client/rpc-spec-v2/src/common/events.rs +++ b/substrate/client/rpc-spec-v2/src/common/events.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; -/// The storage item received as paramter. +/// The storage item to query. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct StorageQuery { @@ -31,6 +31,21 @@ pub struct StorageQuery { pub query_type: StorageQueryType, } +/// The storage item to query with pagination. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PaginatedStorageQuery { + /// The provided key. + pub key: Key, + /// The type of the storage query. + #[serde(rename = "type")] + pub query_type: StorageQueryType, + /// The pagination key from which the iteration should resume. + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub pagination_start_key: Option, +} + /// The type of the storage query. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -165,4 +180,36 @@ mod tests { let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); assert_eq!(dec, item); } + + #[test] + fn storage_query_paginated() { + let item = PaginatedStorageQuery { + key: "0x1", + query_type: StorageQueryType::Value, + pagination_start_key: None, + }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","type":"value"}"#; + assert_eq!(ser, exp); + // Decode + let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap(); + assert_eq!(dec.key, item.key); + assert_eq!(dec.query_type, item.query_type); + let dec: PaginatedStorageQuery<&str> = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + + let item = PaginatedStorageQuery { + key: "0x1", + query_type: StorageQueryType::Value, + pagination_start_key: Some("0x2"), + }; + // Encode + let ser = serde_json::to_string(&item).unwrap(); + let exp = r#"{"key":"0x1","type":"value","paginationStartKey":"0x2"}"#; + assert_eq!(ser, exp); + // Decode + let dec: PaginatedStorageQuery<&str> = serde_json::from_str(exp).unwrap(); + assert_eq!(dec, item); + } } From cf050f7d1f4396cde56c167cda2d68166b206010 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Thu, 5 Oct 2023 11:41:05 +0200 Subject: [PATCH 07/22] archive/storage: Implement archive_storage Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/archive/api.rs | 18 ++- .../client/rpc-spec-v2/src/archive/archive.rs | 49 +++++- .../src/archive/archive_storage.rs | 141 ++++++++++++++++++ .../client/rpc-spec-v2/src/archive/mod.rs | 2 + .../client/rpc-spec-v2/src/archive/tests.rs | 2 +- .../rpc-spec-v2/src/chain_head/event.rs | 2 + .../rpc-spec-v2/src/chain_head/tests.rs | 7 +- .../client/rpc-spec-v2/src/common/events.rs | 39 +++++ substrate/client/rpc-spec-v2/src/lib.rs | 2 +- 9 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 substrate/client/rpc-spec-v2/src/archive/archive_storage.rs diff --git a/substrate/client/rpc-spec-v2/src/archive/api.rs b/substrate/client/rpc-spec-v2/src/archive/api.rs index 0583111cb488..b19738304000 100644 --- a/substrate/client/rpc-spec-v2/src/archive/api.rs +++ b/substrate/client/rpc-spec-v2/src/archive/api.rs @@ -18,7 +18,10 @@ //! API trait of the archive methods. -use crate::MethodResult; +use crate::{ + common::events::{ArchiveStorageResult, PaginatedStorageQuery}, + MethodResult, +}; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; #[rpc(client, server)] @@ -88,4 +91,17 @@ pub trait ArchiveApi { function: String, call_parameters: String, ) -> RpcResult; + + /// Returns storage entries at a specific block's state. + /// + /// # Unstable + /// + /// This method is unstable and subject to change in the future. + #[method(name = "archive_unstable_storage", blocking)] + fn archive_unstable_storage( + &self, + hash: Hash, + items: Vec>, + child_trie: Option, + ) -> RpcResult; } diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs index 5e3f640c7257..f462e98f68e5 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -20,13 +20,15 @@ use crate::{ archive::{error::Error as ArchiveError, ArchiveApiServer}, + common::events::{ArchiveStorageResult, PaginatedStorageQuery, StorageQueryType}, hex_string, MethodResult, }; use codec::Encode; use jsonrpsee::core::{async_trait, RpcResult}; use sc_client_api::{ - Backend, BlockBackend, BlockchainEvents, CallExecutor, ExecutorProvider, StorageProvider, + Backend, BlockBackend, BlockchainEvents, CallExecutor, ChildInfo, ExecutorProvider, StorageKey, + StorageProvider, }; use sp_api::{CallApiAt, CallContext, NumberFor}; use sp_blockchain::{ @@ -39,6 +41,8 @@ use sp_runtime::{ }; use std::{collections::HashSet, marker::PhantomData, sync::Arc}; +use super::archive_storage::ArchiveStorage; + /// An API for archive RPC calls. pub struct Archive, Block: BlockT, Client> { /// Substrate client. @@ -184,4 +188,47 @@ where Err(error) => MethodResult::err(error.to_string()), }) } + + fn archive_unstable_storage( + &self, + hash: Block::Hash, + items: Vec>, + child_trie: Option, + ) -> RpcResult { + let items = items + .into_iter() + .map(|query| { + let key = StorageKey(parse_hex_param(query.key)?); + let pagination_start_key = query + .pagination_start_key + .map(|key| parse_hex_param(key).map(|key| StorageKey(key))) + .transpose()?; + + // Paginated start key is only supported + if pagination_start_key.is_some() && + (query.query_type != StorageQueryType::DescendantsValues || + query.query_type != StorageQueryType::DescendantsHashes) + { + return Err(ArchiveError::InvalidParam( + "Pagination start key is only supported for descendants queries" + .to_string(), + )) + } + + Ok(PaginatedStorageQuery { + key, + query_type: query.query_type, + pagination_start_key, + }) + }) + .collect::, ArchiveError>>()?; + + let child_trie = child_trie + .map(|child_trie| parse_hex_param(child_trie)) + .transpose()? + .map(ChildInfo::new_default_from_vec); + + let storage_client = ArchiveStorage::new(self.client.clone(), 5); + Ok(storage_client.handle_query(hash, items, child_trie)) + } } diff --git a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs new file mode 100644 index 000000000000..6e3641c04700 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs @@ -0,0 +1,141 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Implementation of the `archive_storage` method. + +use std::{marker::PhantomData, sync::Arc}; + +use sc_client_api::{Backend, ChildInfo, StorageKey, StorageProvider}; +use sp_api::BlockT; + +use crate::common::{ + events::{ + ArchiveStorageMethodErr, ArchiveStorageMethodOk, ArchiveStorageResult, + PaginatedStorageQuery, StorageQueryType, + }, + storage::{is_key_queryable, IterQueryType, QueryIter, Storage}, +}; + +/// Generates the events of the `chainHead_storage` method. +pub struct ArchiveStorage { + /// Storage client. + client: Storage, + /// The maximum number of items reported by the `archive_storage`. + operation_max_storage_items: usize, + _phantom: PhantomData<(BE, Block)>, +} + +impl ArchiveStorage { + /// Constructs a new [`ArchiveStorage`]. + pub fn new(client: Arc, operation_max_storage_items: usize) -> Self { + Self { client: Storage::new(client), operation_max_storage_items, _phantom: PhantomData } + } +} + +impl ArchiveStorage +where + Block: BlockT + 'static, + BE: Backend + 'static, + Client: StorageProvider + 'static, +{ + /// Generate the response of the `archive_storage` method. + pub fn handle_query( + &self, + hash: Block::Hash, + items: Vec>, + child_key: Option, + ) -> ArchiveStorageResult { + if let Some(child_key) = child_key.as_ref() { + if !is_key_queryable(child_key.storage_key()) { + return ArchiveStorageResult::Ok(ArchiveStorageMethodOk { + result: Vec::new(), + discarded_items: 0, + }) + } + } + + let mut storage_results = Vec::with_capacity(items.len()); + for item in items { + if !is_key_queryable(&item.key.0) { + continue + } + + match item.query_type { + StorageQueryType::Value => { + match self.client.query_value(hash, &item.key, child_key.as_ref()) { + Ok(Some(value)) => storage_results.push(value), + Ok(None) => continue, + Err(error) => + return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + } + }, + StorageQueryType::Hash => + match self.client.query_hash(hash, &item.key, child_key.as_ref()) { + Ok(Some(value)) => storage_results.push(value), + Ok(None) => continue, + Err(error) => + return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + }, + StorageQueryType::ClosestDescendantMerkleValue => + match self.client.query_merkle_value(hash, &item.key, child_key.as_ref()) { + Ok(Some(value)) => storage_results.push(value), + Ok(None) => continue, + Err(error) => + return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + }, + StorageQueryType::DescendantsValues => { + match self.client.query_iter_pagination( + QueryIter { + query_key: item.key, + ty: IterQueryType::Value, + pagination_start_key: item.pagination_start_key, + }, + hash, + child_key.as_ref(), + self.operation_max_storage_items, + ) { + Ok((results, _)) => storage_results.extend(results), + Err(error) => + return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + } + }, + StorageQueryType::DescendantsHashes => { + match self.client.query_iter_pagination( + QueryIter { + query_key: item.key, + ty: IterQueryType::Hash, + pagination_start_key: item.pagination_start_key, + }, + hash, + child_key.as_ref(), + self.operation_max_storage_items, + ) { + Ok((results, _)) => storage_results.extend(results), + Err(error) => + return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + } + }, + }; + } + + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { + result: storage_results, + discarded_items: 0, + }) + } +} diff --git a/substrate/client/rpc-spec-v2/src/archive/mod.rs b/substrate/client/rpc-spec-v2/src/archive/mod.rs index eb7d71d702f6..e1f45e19a62f 100644 --- a/substrate/client/rpc-spec-v2/src/archive/mod.rs +++ b/substrate/client/rpc-spec-v2/src/archive/mod.rs @@ -25,6 +25,8 @@ #[cfg(test)] mod tests; +mod archive_storage; + pub mod api; pub mod archive; pub mod error; diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs index 36f7716e393c..a3a661e5b89d 100644 --- a/substrate/client/rpc-spec-v2/src/archive/tests.rs +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{chain_head::hex_string, MethodResult}; +use crate::{hex_string, MethodResult}; use super::{archive::Archive, *}; diff --git a/substrate/client/rpc-spec-v2/src/chain_head/event.rs b/substrate/client/rpc-spec-v2/src/chain_head/event.rs index 2de06bacb52a..560ab87eab40 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/event.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/event.rs @@ -340,6 +340,8 @@ pub struct MethodResponseStarted { #[cfg(test)] mod tests { + use crate::common::events::StorageResultType; + use super::*; #[test] diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 8aaeb413cdff..d2bd66763992 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -16,9 +16,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::chain_head::{ - event::{MethodResponse, StorageQuery, StorageQueryType, StorageResultType}, - test_utils::ChainHeadMockClient, +use crate::{ + chain_head::{event::MethodResponse, test_utils::ChainHeadMockClient}, + common::events::{StorageQuery, StorageQueryType, StorageResultType}, + hex_string, }; use super::*; diff --git a/substrate/client/rpc-spec-v2/src/common/events.rs b/substrate/client/rpc-spec-v2/src/common/events.rs index fa208427dccb..207ae618887c 100644 --- a/substrate/client/rpc-spec-v2/src/common/events.rs +++ b/substrate/client/rpc-spec-v2/src/common/events.rs @@ -85,6 +85,45 @@ pub enum StorageResultType { ClosestDescendantMerkleValue(String), } +/// The error of a storage call. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageResultErr { + /// The hex-encoded key of the result. + pub key: String, + /// The result of the query. + #[serde(flatten)] + pub error: StorageResultType, +} + +/// The result of a storage call. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ArchiveStorageResult { + /// Query generated a result. + Ok(ArchiveStorageMethodOk), + /// Query encountered an error. + Err(ArchiveStorageMethodErr), +} + +/// The result of a storage call. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArchiveStorageMethodOk { + /// Reported results. + pub result: Vec, + /// Number of discarded items. + pub discarded_items: usize, +} + +/// The error of a storage call. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArchiveStorageMethodErr { + /// Reported error. + pub error: String, +} + #[cfg(test)] mod tests { use super::*; diff --git a/substrate/client/rpc-spec-v2/src/lib.rs b/substrate/client/rpc-spec-v2/src/lib.rs index 9b851443a2d9..2381fe4431d4 100644 --- a/substrate/client/rpc-spec-v2/src/lib.rs +++ b/substrate/client/rpc-spec-v2/src/lib.rs @@ -42,7 +42,7 @@ pub type SubscriptionTaskExecutor = std::sync::Arc Date: Thu, 5 Oct 2023 11:50:58 +0200 Subject: [PATCH 08/22] archive/storage: Add configurable pagination support Signed-off-by: Alexandru Vasile --- substrate/client/rpc-spec-v2/src/archive/archive.rs | 9 +++++++-- substrate/client/rpc-spec-v2/src/archive/tests.rs | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs index f462e98f68e5..3d6409e0f0dc 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -51,6 +51,9 @@ pub struct Archive, Block: BlockT, Client> { backend: Arc, /// The hexadecimal encoded hash of the genesis block. genesis_hash: String, + /// The maximum number of items reported by the `archive_storage` before + /// pagination is required. + operation_max_storage_items: usize, /// Phantom member to pin the block type. _phantom: PhantomData<(Block, BE)>, } @@ -61,9 +64,10 @@ impl, Block: BlockT, Client> Archive { client: Arc, backend: Arc, genesis_hash: GenesisHash, + operation_max_storage_items: usize, ) -> Self { let genesis_hash = hex_string(&genesis_hash.as_ref()); - Self { client, backend, genesis_hash, _phantom: PhantomData } + Self { client, backend, genesis_hash, operation_max_storage_items, _phantom: PhantomData } } } @@ -228,7 +232,8 @@ where .transpose()? .map(ChildInfo::new_default_from_vec); - let storage_client = ArchiveStorage::new(self.client.clone(), 5); + let storage_client = + ArchiveStorage::new(self.client.clone(), self.operation_max_storage_items); Ok(storage_client.handle_query(hash, items, child_trie)) } } diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs index a3a661e5b89d..1127dc61d1d0 100644 --- a/substrate/client/rpc-spec-v2/src/archive/tests.rs +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -39,6 +39,7 @@ use substrate_test_runtime_client::{ const CHAIN_GENESIS: [u8; 32] = [0; 32]; const INVALID_HASH: [u8; 32] = [1; 32]; +const MAX_PAGINATION_LIMIT: usize = 5; type Header = substrate_test_runtime_client::runtime::Header; type Block = substrate_test_runtime_client::runtime::Block; @@ -48,7 +49,7 @@ fn setup_api() -> (Arc>, RpcModule Date: Fri, 6 Oct 2023 10:41:03 +0200 Subject: [PATCH 09/22] archive/tests: Check hashes and values Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/archive/tests.rs | 86 ++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs index 1127dc61d1d0..821b76ee7770 100644 --- a/substrate/client/rpc-spec-v2/src/archive/tests.rs +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -16,7 +16,13 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{hex_string, MethodResult}; +use crate::{ + common::events::{ + ArchiveStorageMethodOk, ArchiveStorageResult, PaginatedStorageQuery, StorageQueryType, + StorageResultType, + }, + hex_string, MethodResult, +}; use super::{archive::Archive, *}; @@ -24,12 +30,15 @@ use assert_matches::assert_matches; use codec::{Decode, Encode}; use jsonrpsee::{ core::error::Error, + rpc_params, types::{error::CallError, EmptyServerParams as EmptyParams}, RpcModule, }; use sc_block_builder::BlockBuilderProvider; +use sc_client_api::ChildInfo; use sp_blockchain::HeaderBackend; use sp_consensus::BlockOrigin; +use sp_core::{Blake2Hasher, Hasher}; use sp_runtime::SaturatedConversion; use std::sync::Arc; use substrate_test_runtime::Transfer; @@ -40,12 +49,21 @@ use substrate_test_runtime_client::{ const CHAIN_GENESIS: [u8; 32] = [0; 32]; const INVALID_HASH: [u8; 32] = [1; 32]; const MAX_PAGINATION_LIMIT: usize = 5; +const KEY: &[u8] = b":mock"; +const VALUE: &[u8] = b"hello world"; +const CHILD_STORAGE_KEY: &[u8] = b"child"; +const CHILD_VALUE: &[u8] = b"child value"; type Header = substrate_test_runtime_client::runtime::Header; type Block = substrate_test_runtime_client::runtime::Block; fn setup_api() -> (Arc>, RpcModule>>) { - let builder = TestClientBuilder::new(); + let child_info = ChildInfo::new_default(CHILD_STORAGE_KEY); + let builder = TestClientBuilder::new().add_extra_child_storage( + &child_info, + KEY.to_vec(), + CHILD_VALUE.to_vec(), + ); let backend = builder.backend(); let client = Arc::new(builder.build()); @@ -257,3 +275,67 @@ async fn archive_call() { let expected = MethodResult::ok("0x0000000000000000"); assert_eq!(result, expected); } + +#[tokio::test] +async fn archive_storage_hashes_values() { + let (mut client, api) = setup_api(); + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + let block_hash = format!("{:?}", block.header.hash()); + let key = hex_string(&KEY); + + let items: Vec> = vec![ + PaginatedStorageQuery { + key: key.clone(), + query_type: StorageQueryType::DescendantsHashes, + pagination_start_key: None, + }, + PaginatedStorageQuery { + key: key.clone(), + query_type: StorageQueryType::DescendantsValues, + pagination_start_key: None, + }, + ]; + + let result: ArchiveStorageResult = api + .call("archive_unstable_storage", rpc_params![&block_hash, items.clone()]) + .await + .unwrap(); + + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + // Key has not been imported yet. + assert_eq!(result.len(), 0); + assert_eq!(discarded_items, 0); + }, + _ => panic!("Unexpected result"), + }; + + // Import a block with the given key value pair. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(KEY.to_vec(), Some(VALUE.to_vec())).unwrap(); + let block = builder.build().unwrap().block; + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + let block_hash = format!("{:?}", block.header.hash()); + let expected_hash = format!("{:?}", Blake2Hasher::hash(&VALUE)); + let expected_value = hex_string(&VALUE); + + let result: ArchiveStorageResult = api + .call("archive_unstable_storage", rpc_params![&block_hash, items]) + .await + .unwrap(); + + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + assert_eq!(result.len(), 2); + assert_eq!(discarded_items, 0); + + assert_eq!(result[0].key, key); + assert_eq!(result[0].result, StorageResultType::Hash(expected_hash)); + assert_eq!(result[1].key, key); + assert_eq!(result[1].result, StorageResultType::Value(expected_value)); + }, + _ => panic!("Unexpected result"), + }; +} From 1cde37d9d6f2db78956f8d6af3ef5ab0aa52cbb6 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 6 Oct 2023 10:50:03 +0200 Subject: [PATCH 10/22] archive/tests: Test merkle value returns the expected hashes Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/archive/tests.rs | 154 +++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs index 821b76ee7770..3a5a0d4987f0 100644 --- a/substrate/client/rpc-spec-v2/src/archive/tests.rs +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -40,7 +40,7 @@ use sp_blockchain::HeaderBackend; use sp_consensus::BlockOrigin; use sp_core::{Blake2Hasher, Hasher}; use sp_runtime::SaturatedConversion; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use substrate_test_runtime::Transfer; use substrate_test_runtime_client::{ prelude::*, runtime, Backend, BlockBuilderExt, Client, ClientBlockImportExt, @@ -339,3 +339,155 @@ async fn archive_storage_hashes_values() { _ => panic!("Unexpected result"), }; } + +#[tokio::test] +async fn archive_storage_closest_merkle_value() { + let (mut client, api) = setup_api(); + + /// The core of this test. + /// + /// Checks keys that are exact match, keys with descedant and keys that should not return + /// values. + /// + /// Returns (key, merkle value) pairs. + async fn expect_merkle_request( + api: &RpcModule>>, + block_hash: String, + ) -> HashMap { + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &block_hash, + vec![ + PaginatedStorageQuery { + key: hex_string(b":AAAA"), + query_type: StorageQueryType::ClosestDescendantMerkleValue, + pagination_start_key: None, + }, + PaginatedStorageQuery { + key: hex_string(b":AAAB"), + query_type: StorageQueryType::ClosestDescendantMerkleValue, + pagination_start_key: None, + }, + // Key with descedent. + PaginatedStorageQuery { + key: hex_string(b":A"), + query_type: StorageQueryType::ClosestDescendantMerkleValue, + pagination_start_key: None, + }, + PaginatedStorageQuery { + key: hex_string(b":AA"), + query_type: StorageQueryType::ClosestDescendantMerkleValue, + pagination_start_key: None, + }, + // Keys below this comment do not produce a result. + // Key that exceed the keyspace of the trie. + PaginatedStorageQuery { + key: hex_string(b":AAAAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue, + pagination_start_key: None, + }, + PaginatedStorageQuery { + key: hex_string(b":AAABX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue, + pagination_start_key: None, + }, + // Key that are not part of the trie. + PaginatedStorageQuery { + key: hex_string(b":AAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue, + pagination_start_key: None, + }, + PaginatedStorageQuery { + key: hex_string(b":AAAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue, + pagination_start_key: None, + }, + ] + ], + ) + .await + .unwrap(); + + let merkle_values: HashMap<_, _> = match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, .. }) => result + .into_iter() + .map(|res| { + let value = match res.result { + StorageResultType::ClosestDescendantMerkleValue(value) => value, + _ => panic!("Unexpected StorageResultType"), + }; + (res.key, value) + }) + .collect(), + _ => panic!("Unexpected result"), + }; + + // Response for AAAA, AAAB, A and AA. + assert_eq!(merkle_values.len(), 4); + + // While checking for expected merkle values to align, + // the following will check that the returned keys are + // expected. + + // Values for AAAA and AAAB are different. + assert_ne!( + merkle_values.get(&hex_string(b":AAAA")).unwrap(), + merkle_values.get(&hex_string(b":AAAB")).unwrap() + ); + + // Values for A and AA should be on the same branch node. + assert_eq!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AA")).unwrap() + ); + // The branch node value must be different than the leaf of either + // AAAA and AAAB. + assert_ne!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AAAA")).unwrap() + ); + assert_ne!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AAAB")).unwrap() + ); + + merkle_values + } + + // Import a new block with storage changes. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(b":AAAA".to_vec(), Some(vec![1; 64])).unwrap(); + builder.push_storage_change(b":AAAB".to_vec(), Some(vec![2; 64])).unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + let merkle_values_lhs = expect_merkle_request(&api, block_hash).await; + + // Import a new block with and change AAAB value. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(b":AAAA".to_vec(), Some(vec![1; 64])).unwrap(); + builder.push_storage_change(b":AAAB".to_vec(), Some(vec![3; 64])).unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + let merkle_values_rhs = expect_merkle_request(&api, block_hash).await; + + // Change propagated to the root. + assert_ne!( + merkle_values_lhs.get(&hex_string(b":A")).unwrap(), + merkle_values_rhs.get(&hex_string(b":A")).unwrap() + ); + assert_ne!( + merkle_values_lhs.get(&hex_string(b":AAAB")).unwrap(), + merkle_values_rhs.get(&hex_string(b":AAAB")).unwrap() + ); + // However the AAAA branch leaf remains unchanged. + assert_eq!( + merkle_values_lhs.get(&hex_string(b":AAAA")).unwrap(), + merkle_values_rhs.get(&hex_string(b":AAAA")).unwrap() + ); +} From d560bb75be3ad63db20d6180e8b4aabf0d0f74dc Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 6 Oct 2023 11:08:46 +0200 Subject: [PATCH 11/22] archive/tests: Check pagination support produces variable items Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/archive/archive.rs | 2 +- .../client/rpc-spec-v2/src/archive/tests.rs | 213 +++++++++++++++++- 2 files changed, 204 insertions(+), 11 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs index 3d6409e0f0dc..21d12db3f0f4 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -210,7 +210,7 @@ where // Paginated start key is only supported if pagination_start_key.is_some() && - (query.query_type != StorageQueryType::DescendantsValues || + (query.query_type != StorageQueryType::DescendantsValues && query.query_type != StorageQueryType::DescendantsHashes) { return Err(ArchiveError::InvalidParam( diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs index 3a5a0d4987f0..87f3de213afd 100644 --- a/substrate/client/rpc-spec-v2/src/archive/tests.rs +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -57,7 +57,9 @@ const CHILD_VALUE: &[u8] = b"child value"; type Header = substrate_test_runtime_client::runtime::Header; type Block = substrate_test_runtime_client::runtime::Block; -fn setup_api() -> (Arc>, RpcModule>>) { +fn setup_api( + ops: usize, +) -> (Arc>, RpcModule>>) { let child_info = ChildInfo::new_default(CHILD_STORAGE_KEY); let builder = TestClientBuilder::new().add_extra_child_storage( &child_info, @@ -67,14 +69,14 @@ fn setup_api() -> (Arc>, RpcModule = api.call("archive_unstable_hashByHeight", [0]).await.unwrap(); @@ -226,7 +228,7 @@ async fn archive_hash_by_height() { #[tokio::test] async fn archive_call() { - let (mut client, api) = setup_api(); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); let invalid_hash = hex_string(&INVALID_HASH); // Invalid parameter (non-hex). @@ -278,7 +280,7 @@ async fn archive_call() { #[tokio::test] async fn archive_storage_hashes_values() { - let (mut client, api) = setup_api(); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; client.import(BlockOrigin::Own, block.clone()).await.unwrap(); let block_hash = format!("{:?}", block.header.hash()); @@ -342,7 +344,7 @@ async fn archive_storage_hashes_values() { #[tokio::test] async fn archive_storage_closest_merkle_value() { - let (mut client, api) = setup_api(); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); /// The core of this test. /// @@ -491,3 +493,194 @@ async fn archive_storage_closest_merkle_value() { merkle_values_rhs.get(&hex_string(b":AAAA")).unwrap() ); } + +#[tokio::test] +async fn archive_storage_paginate_iterations() { + // 1 iteration allowed before pagination kicks in. + let (mut client, api) = setup_api(1); + + // Import a new block with storage changes. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(b":m".to_vec(), Some(b"a".to_vec())).unwrap(); + builder.push_storage_change(b":mo".to_vec(), Some(b"ab".to_vec())).unwrap(); + builder.push_storage_change(b":moc".to_vec(), Some(b"abc".to_vec())).unwrap(); + builder.push_storage_change(b":moD".to_vec(), Some(b"abcmoD".to_vec())).unwrap(); + builder.push_storage_change(b":mock".to_vec(), Some(b"abcd".to_vec())).unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + // Calling with an invalid hash. + let invalid_hash = hex_string(&INVALID_HASH); + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &invalid_hash, + vec![PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::DescendantsValues, + pagination_start_key: None, + }] + ], + ) + .await + .unwrap(); + match result { + ArchiveStorageResult::Err(_) => (), + _ => panic!("Unexpected result"), + }; + + // Valid call with storage at the key. + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &block_hash, + vec![PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::DescendantsValues, + pagination_start_key: None, + }] + ], + ) + .await + .unwrap(); + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + assert_eq!(result.len(), 1); + assert_eq!(discarded_items, 0); + + assert_eq!(result[0].key, hex_string(b":m")); + assert_eq!(result[0].result, StorageResultType::Value(hex_string(b"a"))); + }, + _ => panic!("Unexpected result"), + }; + + // Continue with pagination. + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &block_hash, + vec![PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::DescendantsValues, + pagination_start_key: Some(hex_string(b":m")), + }] + ], + ) + .await + .unwrap(); + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + assert_eq!(result.len(), 1); + assert_eq!(discarded_items, 0); + + assert_eq!(result[0].key, hex_string(b":mo")); + assert_eq!(result[0].result, StorageResultType::Value(hex_string(b"ab"))); + }, + _ => panic!("Unexpected result"), + }; + + // Continue with pagination. + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &block_hash, + vec![PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::DescendantsValues, + pagination_start_key: Some(hex_string(b":mo")), + }] + ], + ) + .await + .unwrap(); + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + assert_eq!(result.len(), 1); + assert_eq!(discarded_items, 0); + + assert_eq!(result[0].key, hex_string(b":moD")); + assert_eq!(result[0].result, StorageResultType::Value(hex_string(b"abcmoD"))); + }, + _ => panic!("Unexpected result"), + }; + + // Continue with pagination. + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &block_hash, + vec![PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::DescendantsValues, + pagination_start_key: Some(hex_string(b":moD")), + }] + ], + ) + .await + .unwrap(); + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + assert_eq!(result.len(), 1); + assert_eq!(discarded_items, 0); + + assert_eq!(result[0].key, hex_string(b":moc")); + assert_eq!(result[0].result, StorageResultType::Value(hex_string(b"abc"))); + }, + _ => panic!("Unexpected result"), + }; + + // Continue with pagination. + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &block_hash, + vec![PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::DescendantsValues, + pagination_start_key: Some(hex_string(b":moc")), + }] + ], + ) + .await + .unwrap(); + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + assert_eq!(result.len(), 1); + assert_eq!(discarded_items, 0); + + assert_eq!(result[0].key, hex_string(b":mock")); + assert_eq!(result[0].result, StorageResultType::Value(hex_string(b"abcd"))); + }, + _ => panic!("Unexpected result"), + }; + + // Continue with pagination until no keys are returned. + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &block_hash, + vec![PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::DescendantsValues, + pagination_start_key: Some(hex_string(b":mock")), + }] + ], + ) + .await + .unwrap(); + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + assert_eq!(result.len(), 0); + assert_eq!(discarded_items, 0); + }, + _ => panic!("Unexpected result"), + }; +} From f29b7619d7aa26c4503faa4ecde29b8ba6a94aea Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Wed, 11 Oct 2023 12:08:29 +0200 Subject: [PATCH 12/22] archive/tests: Check hash and value queries Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/archive/tests.rs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs index 87f3de213afd..bd58e516c743 100644 --- a/substrate/client/rpc-spec-v2/src/archive/tests.rs +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -297,6 +297,16 @@ async fn archive_storage_hashes_values() { query_type: StorageQueryType::DescendantsValues, pagination_start_key: None, }, + PaginatedStorageQuery { + key: key.clone(), + query_type: StorageQueryType::Hash, + pagination_start_key: None, + }, + PaginatedStorageQuery { + key: key.clone(), + query_type: StorageQueryType::Value, + pagination_start_key: None, + }, ]; let result: ArchiveStorageResult = api @@ -330,13 +340,17 @@ async fn archive_storage_hashes_values() { match result { ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { - assert_eq!(result.len(), 2); + assert_eq!(result.len(), 4); assert_eq!(discarded_items, 0); assert_eq!(result[0].key, key); - assert_eq!(result[0].result, StorageResultType::Hash(expected_hash)); + assert_eq!(result[0].result, StorageResultType::Hash(expected_hash.clone())); assert_eq!(result[1].key, key); - assert_eq!(result[1].result, StorageResultType::Value(expected_value)); + assert_eq!(result[1].result, StorageResultType::Value(expected_value.clone())); + assert_eq!(result[2].key, key); + assert_eq!(result[2].result, StorageResultType::Hash(expected_hash)); + assert_eq!(result[3].key, key); + assert_eq!(result[3].result, StorageResultType::Value(expected_value)); }, _ => panic!("Unexpected result"), }; From 59146603177c76b887c78cd19dac17813f6f1927 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Wed, 11 Oct 2023 12:34:10 +0200 Subject: [PATCH 13/22] archive: Add maximum number of queried items Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/archive/archive.rs | 26 +++++-- .../src/archive/archive_storage.rs | 30 ++++++-- .../client/rpc-spec-v2/src/archive/tests.rs | 77 ++++++++++++++++--- 3 files changed, 107 insertions(+), 26 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs index 21d12db3f0f4..fbb8094d1c99 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -51,9 +51,10 @@ pub struct Archive, Block: BlockT, Client> { backend: Arc, /// The hexadecimal encoded hash of the genesis block. genesis_hash: String, - /// The maximum number of items reported by the `archive_storage` before - /// pagination is required. - operation_max_storage_items: usize, + /// The maximum number of reported items by the `archive_storage` at a time. + storage_max_reported_items: usize, + /// The maximum number of queried items allowed for the `archive_storage` at a time. + storage_max_queried_items: usize, /// Phantom member to pin the block type. _phantom: PhantomData<(Block, BE)>, } @@ -64,10 +65,18 @@ impl, Block: BlockT, Client> Archive { client: Arc, backend: Arc, genesis_hash: GenesisHash, - operation_max_storage_items: usize, + storage_max_reported_items: usize, + storage_max_queried_items: usize, ) -> Self { let genesis_hash = hex_string(&genesis_hash.as_ref()); - Self { client, backend, genesis_hash, operation_max_storage_items, _phantom: PhantomData } + Self { + client, + backend, + genesis_hash, + storage_max_reported_items, + storage_max_queried_items, + _phantom: PhantomData, + } } } @@ -232,8 +241,11 @@ where .transpose()? .map(ChildInfo::new_default_from_vec); - let storage_client = - ArchiveStorage::new(self.client.clone(), self.operation_max_storage_items); + let storage_client = ArchiveStorage::new( + self.client.clone(), + self.storage_max_reported_items, + self.storage_max_queried_items, + ); Ok(storage_client.handle_query(hash, items, child_trie)) } } diff --git a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs index 6e3641c04700..690db39e83c2 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs @@ -35,15 +35,26 @@ use crate::common::{ pub struct ArchiveStorage { /// Storage client. client: Storage, - /// The maximum number of items reported by the `archive_storage`. - operation_max_storage_items: usize, + /// The maximum number of reported items by the `archive_storage` at a time. + storage_max_reported_items: usize, + /// The maximum number of queried items allowed for the `archive_storage` at a time. + storage_max_queried_items: usize, _phantom: PhantomData<(BE, Block)>, } impl ArchiveStorage { /// Constructs a new [`ArchiveStorage`]. - pub fn new(client: Arc, operation_max_storage_items: usize) -> Self { - Self { client: Storage::new(client), operation_max_storage_items, _phantom: PhantomData } + pub fn new( + client: Arc, + storage_max_reported_items: usize, + storage_max_queried_items: usize, + ) -> Self { + Self { + client: Storage::new(client), + storage_max_reported_items, + storage_max_queried_items, + _phantom: PhantomData, + } } } @@ -57,7 +68,7 @@ where pub fn handle_query( &self, hash: Block::Hash, - items: Vec>, + mut items: Vec>, child_key: Option, ) -> ArchiveStorageResult { if let Some(child_key) = child_key.as_ref() { @@ -69,6 +80,9 @@ where } } + let discarded_items = items.len().saturating_sub(self.storage_max_queried_items); + items.truncate(self.storage_max_queried_items); + let mut storage_results = Vec::with_capacity(items.len()); for item in items { if !is_key_queryable(&item.key.0) { @@ -107,7 +121,7 @@ where }, hash, child_key.as_ref(), - self.operation_max_storage_items, + self.storage_max_reported_items, ) { Ok((results, _)) => storage_results.extend(results), Err(error) => @@ -123,7 +137,7 @@ where }, hash, child_key.as_ref(), - self.operation_max_storage_items, + self.storage_max_reported_items, ) { Ok((results, _)) => storage_results.extend(results), Err(error) => @@ -135,7 +149,7 @@ where ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result: storage_results, - discarded_items: 0, + discarded_items, }) } } diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs index bd58e516c743..bf56368e1b96 100644 --- a/substrate/client/rpc-spec-v2/src/archive/tests.rs +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -49,6 +49,7 @@ use substrate_test_runtime_client::{ const CHAIN_GENESIS: [u8; 32] = [0; 32]; const INVALID_HASH: [u8; 32] = [1; 32]; const MAX_PAGINATION_LIMIT: usize = 5; +const MAX_QUERIED_LIMIT: usize = 5; const KEY: &[u8] = b":mock"; const VALUE: &[u8] = b"hello world"; const CHILD_STORAGE_KEY: &[u8] = b"child"; @@ -58,7 +59,8 @@ type Header = substrate_test_runtime_client::runtime::Header; type Block = substrate_test_runtime_client::runtime::Block; fn setup_api( - ops: usize, + max_returned_items: usize, + max_queried_items: usize, ) -> (Arc>, RpcModule>>) { let child_info = ChildInfo::new_default(CHILD_STORAGE_KEY); let builder = TestClientBuilder::new().add_extra_child_storage( @@ -69,14 +71,16 @@ fn setup_api( let backend = builder.backend(); let client = Arc::new(builder.build()); - let api = Archive::new(client.clone(), backend, CHAIN_GENESIS, ops).into_rpc(); + let api = + Archive::new(client.clone(), backend, CHAIN_GENESIS, max_returned_items, max_queried_items) + .into_rpc(); (client, api) } #[tokio::test] async fn archive_genesis() { - let (_client, api) = setup_api(MAX_PAGINATION_LIMIT); + let (_client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); let genesis: String = api.call("archive_unstable_genesisHash", EmptyParams::new()).await.unwrap(); @@ -85,7 +89,7 @@ async fn archive_genesis() { #[tokio::test] async fn archive_body() { - let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); // Invalid block hash. let invalid_hash = hex_string(&INVALID_HASH); @@ -114,7 +118,7 @@ async fn archive_body() { #[tokio::test] async fn archive_header() { - let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); // Invalid block hash. let invalid_hash = hex_string(&INVALID_HASH); @@ -143,7 +147,7 @@ async fn archive_header() { #[tokio::test] async fn archive_finalized_height() { - let (client, api) = setup_api(MAX_PAGINATION_LIMIT); + let (client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); let client_height: u32 = client.info().finalized_number.saturated_into(); @@ -155,7 +159,7 @@ async fn archive_finalized_height() { #[tokio::test] async fn archive_hash_by_height() { - let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); // Genesis height. let hashes: Vec = api.call("archive_unstable_hashByHeight", [0]).await.unwrap(); @@ -228,7 +232,7 @@ async fn archive_hash_by_height() { #[tokio::test] async fn archive_call() { - let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); let invalid_hash = hex_string(&INVALID_HASH); // Invalid parameter (non-hex). @@ -280,7 +284,7 @@ async fn archive_call() { #[tokio::test] async fn archive_storage_hashes_values() { - let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; client.import(BlockOrigin::Own, block.clone()).await.unwrap(); let block_hash = format!("{:?}", block.header.hash()); @@ -358,7 +362,7 @@ async fn archive_storage_hashes_values() { #[tokio::test] async fn archive_storage_closest_merkle_value() { - let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT); + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); /// The core of this test. /// @@ -511,7 +515,7 @@ async fn archive_storage_closest_merkle_value() { #[tokio::test] async fn archive_storage_paginate_iterations() { // 1 iteration allowed before pagination kicks in. - let (mut client, api) = setup_api(1); + let (mut client, api) = setup_api(1, MAX_QUERIED_LIMIT); // Import a new block with storage changes. let mut builder = client.new_block(Default::default()).unwrap(); @@ -698,3 +702,54 @@ async fn archive_storage_paginate_iterations() { _ => panic!("Unexpected result"), }; } + +#[tokio::test] +async fn archive_storage_discarded_items() { + // One query at a time + let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, 1); + + // Import a new block with storage changes. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(b":m".to_vec(), Some(b"a".to_vec())).unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + // Valid call with storage at the key. + let result: ArchiveStorageResult = api + .call( + "archive_unstable_storage", + rpc_params![ + &block_hash, + vec![ + PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::Value, + pagination_start_key: None, + }, + PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::Hash, + pagination_start_key: None, + }, + PaginatedStorageQuery { + key: hex_string(b":m"), + query_type: StorageQueryType::Hash, + pagination_start_key: None, + } + ] + ], + ) + .await + .unwrap(); + match result { + ArchiveStorageResult::Ok(ArchiveStorageMethodOk { result, discarded_items }) => { + assert_eq!(result.len(), 1); + assert_eq!(discarded_items, 2); + + assert_eq!(result[0].key, hex_string(b":m")); + assert_eq!(result[0].result, StorageResultType::Value(hex_string(b"a"))); + }, + _ => panic!("Unexpected result"), + }; +} From f4f1b23f055d186db8e6c80a7271ca5be0efdd39 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Tue, 5 Dec 2023 16:18:15 +0200 Subject: [PATCH 14/22] archive/tests: Port tests to BlockBuilder Signed-off-by: Alexandru Vasile --- .../src/archive/archive_storage.rs | 2 +- .../client/rpc-spec-v2/src/archive/tests.rs | 43 ++++++++++++++++--- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs index 395d6c201eef..4a2287fc7a32 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs @@ -18,7 +18,7 @@ //! Implementation of the `archive_storage` method. -use std::{marker::PhantomData, sync::Arc}; +use std::sync::Arc; use sc_client_api::{Backend, ChildInfo, StorageKey, StorageProvider}; use sp_runtime::traits::Block as BlockT; diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs index d9efeb8910bd..45da8e588e62 100644 --- a/substrate/client/rpc-spec-v2/src/archive/tests.rs +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -38,11 +38,12 @@ use sc_block_builder::BlockBuilderBuilder; use sc_client_api::ChildInfo; use sp_blockchain::HeaderBackend; use sp_consensus::BlockOrigin; +use sp_core::{Blake2Hasher, Hasher}; use sp_runtime::{ traits::{Block as BlockT, Header as HeaderT}, SaturatedConversion, }; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use substrate_test_runtime::Transfer; use substrate_test_runtime_client::{ prelude::*, runtime, Backend, BlockBuilderExt, Client, ClientBlockImportExt, @@ -337,7 +338,15 @@ async fn archive_call() { #[tokio::test] async fn archive_storage_hashes_values() { let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, MAX_QUERIED_LIMIT); - let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + + let block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().genesis_hash) + .with_parent_block_number(0) + .build() + .unwrap() + .build() + .unwrap() + .block; client.import(BlockOrigin::Own, block.clone()).await.unwrap(); let block_hash = format!("{:?}", block.header.hash()); let key = hex_string(&KEY); @@ -380,7 +389,11 @@ async fn archive_storage_hashes_values() { }; // Import a block with the given key value pair. - let mut builder = client.new_block(Default::default()).unwrap(); + let mut builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(block.hash()) + .with_parent_block_number(1) + .build() + .unwrap(); builder.push_storage_change(KEY.to_vec(), Some(VALUE.to_vec())).unwrap(); let block = builder.build().unwrap().block; client.import(BlockOrigin::Own, block.clone()).await.unwrap(); @@ -529,7 +542,11 @@ async fn archive_storage_closest_merkle_value() { } // Import a new block with storage changes. - let mut builder = client.new_block(Default::default()).unwrap(); + let mut builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().genesis_hash) + .with_parent_block_number(0) + .build() + .unwrap(); builder.push_storage_change(b":AAAA".to_vec(), Some(vec![1; 64])).unwrap(); builder.push_storage_change(b":AAAB".to_vec(), Some(vec![2; 64])).unwrap(); let block = builder.build().unwrap().block; @@ -539,7 +556,11 @@ async fn archive_storage_closest_merkle_value() { let merkle_values_lhs = expect_merkle_request(&api, block_hash).await; // Import a new block with and change AAAB value. - let mut builder = client.new_block(Default::default()).unwrap(); + let mut builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(block.hash()) + .with_parent_block_number(1) + .build() + .unwrap(); builder.push_storage_change(b":AAAA".to_vec(), Some(vec![1; 64])).unwrap(); builder.push_storage_change(b":AAAB".to_vec(), Some(vec![3; 64])).unwrap(); let block = builder.build().unwrap().block; @@ -570,7 +591,11 @@ async fn archive_storage_paginate_iterations() { let (mut client, api) = setup_api(1, MAX_QUERIED_LIMIT); // Import a new block with storage changes. - let mut builder = client.new_block(Default::default()).unwrap(); + let mut builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().genesis_hash) + .with_parent_block_number(0) + .build() + .unwrap(); builder.push_storage_change(b":m".to_vec(), Some(b"a".to_vec())).unwrap(); builder.push_storage_change(b":mo".to_vec(), Some(b"ab".to_vec())).unwrap(); builder.push_storage_change(b":moc".to_vec(), Some(b"abc".to_vec())).unwrap(); @@ -761,7 +786,11 @@ async fn archive_storage_discarded_items() { let (mut client, api) = setup_api(MAX_PAGINATION_LIMIT, 1); // Import a new block with storage changes. - let mut builder = client.new_block(Default::default()).unwrap(); + let mut builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().genesis_hash) + .with_parent_block_number(0) + .build() + .unwrap(); builder.push_storage_change(b":m".to_vec(), Some(b"a".to_vec())).unwrap(); let block = builder.build().unwrap().block; let block_hash = format!("{:?}", block.header.hash()); From 1de7eb2378087f0f30d596bda371dd9ff133bb8d Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Thu, 7 Dec 2023 15:02:55 +0200 Subject: [PATCH 15/22] spec-v2: Remove the concept of non-queriable keys Signed-off-by: Alexandru Vasile --- .../rpc-spec-v2/src/archive/archive_storage.rs | 15 +-------------- .../src/chain_head/chain_head_storage.rs | 16 +--------------- .../client/rpc-spec-v2/src/common/storage.rs | 10 ---------- 3 files changed, 2 insertions(+), 39 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs index 4a2287fc7a32..351c1aae9a43 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs @@ -28,7 +28,7 @@ use crate::common::{ ArchiveStorageMethodErr, ArchiveStorageMethodOk, ArchiveStorageResult, PaginatedStorageQuery, StorageQueryType, }, - storage::{is_key_queryable, IterQueryType, QueryIter, Storage}, + storage::{IterQueryType, QueryIter, Storage}, }; /// Generates the events of the `chainHead_storage` method. @@ -65,24 +65,11 @@ where mut items: Vec>, child_key: Option, ) -> ArchiveStorageResult { - if let Some(child_key) = child_key.as_ref() { - if !is_key_queryable(child_key.storage_key()) { - return ArchiveStorageResult::Ok(ArchiveStorageMethodOk { - result: Vec::new(), - discarded_items: 0, - }) - } - } - let discarded_items = items.len().saturating_sub(self.storage_max_queried_items); items.truncate(self.storage_max_queried_items); let mut storage_results = Vec::with_capacity(items.len()); for item in items { - if !is_key_queryable(&item.key.0) { - continue - } - match item.query_type { StorageQueryType::Value => { match self.client.query_value(hash, &item.key, child_key.as_ref()) { diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index efacad722af7..737f64c615fd 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -32,7 +32,7 @@ use crate::{ }, common::{ events::{StorageQuery, StorageQueryType}, - storage::{is_key_queryable, IterQueryType, QueryIter, QueryIterResult, Storage}, + storage::{ IterQueryType, QueryIter, QueryIterResult, Storage}, }, }; @@ -139,22 +139,8 @@ where let sender = block_guard.response_sender(); let operation = block_guard.operation(); - if let Some(child_key) = child_key.as_ref() { - if !is_key_queryable(child_key.storage_key()) { - let _ = sender.unbounded_send(FollowEvent::::OperationStorageDone( - OperationId { operation_id: operation.operation_id() }, - )); - return - } - } - let mut storage_results = Vec::with_capacity(items.len()); for item in items { - if !is_key_queryable(&item.key.0) { - continue - } - - match item.query_type { StorageQueryType::Value => { match self.client.query_value(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), diff --git a/substrate/client/rpc-spec-v2/src/common/storage.rs b/substrate/client/rpc-spec-v2/src/common/storage.rs index 8f8effc41253..e772cc0f9294 100644 --- a/substrate/client/rpc-spec-v2/src/common/storage.rs +++ b/substrate/client/rpc-spec-v2/src/common/storage.rs @@ -59,16 +59,6 @@ pub enum IterQueryType { Hash, } -/// Checks if the provided key (main or child key) is valid -/// for queries. -/// -/// Keys that are identical to `:child_storage:` or `:child_storage:default:` -/// are not queryable. -pub fn is_key_queryable(key: &[u8]) -> bool { - !well_known_keys::is_default_child_storage_key(key) && - !well_known_keys::is_child_storage_key(key) -} - /// The result of making a query call. pub type QueryResult = Result, String>; From dd70064e6ffedb40746f85cc8faba5484cd4cc14 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Thu, 7 Dec 2023 15:05:53 +0200 Subject: [PATCH 16/22] Fix build Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/chain_head/chain_head_storage.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index 737f64c615fd..ee39ec253a30 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -32,7 +32,7 @@ use crate::{ }, common::{ events::{StorageQuery, StorageQueryType}, - storage::{ IterQueryType, QueryIter, QueryIterResult, Storage}, + storage::{IterQueryType, QueryIter, QueryIterResult, Storage}, }, }; @@ -141,6 +141,7 @@ where let mut storage_results = Vec::with_capacity(items.len()); for item in items { + match item.query_type { StorageQueryType::Value => { match self.client.query_value(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), From 8ad951316826c8f25cc302db0ebd9f0de3e697ba Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Thu, 7 Dec 2023 15:25:39 +0200 Subject: [PATCH 17/22] storage: Remove unused imports Signed-off-by: Alexandru Vasile --- substrate/client/rpc-spec-v2/src/common/storage.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/substrate/client/rpc-spec-v2/src/common/storage.rs b/substrate/client/rpc-spec-v2/src/common/storage.rs index e772cc0f9294..bd249e033f8f 100644 --- a/substrate/client/rpc-spec-v2/src/common/storage.rs +++ b/substrate/client/rpc-spec-v2/src/common/storage.rs @@ -21,7 +21,6 @@ use std::{marker::PhantomData, sync::Arc}; use sc_client_api::{Backend, ChildInfo, StorageKey, StorageProvider}; -use sp_core::storage::well_known_keys; use sp_runtime::traits::Block as BlockT; use super::events::{StorageResult, StorageResultType}; From 6f3920ef2b1de9c65b9185a3e7a3e04e10f5266d Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 15 Jan 2024 13:54:41 +0200 Subject: [PATCH 18/22] rpc-v2/events: Add `StorageQueryType::is_descendant_query` method Signed-off-by: Alexandru Vasile --- substrate/client/rpc-spec-v2/src/archive/archive.rs | 5 +---- substrate/client/rpc-spec-v2/src/common/events.rs | 7 +++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs index ea0e2b3f0dbf..b8c9fc825494 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -218,10 +218,7 @@ where .transpose()?; // Paginated start key is only supported - if pagination_start_key.is_some() && - (query.query_type != StorageQueryType::DescendantsValues && - query.query_type != StorageQueryType::DescendantsHashes) - { + if pagination_start_key.is_some() && !query.query_type.is_descendant_query() { return Err(ArchiveError::InvalidParam( "Pagination start key is only supported for descendants queries" .to_string(), diff --git a/substrate/client/rpc-spec-v2/src/common/events.rs b/substrate/client/rpc-spec-v2/src/common/events.rs index 207ae618887c..e30ce4fccc99 100644 --- a/substrate/client/rpc-spec-v2/src/common/events.rs +++ b/substrate/client/rpc-spec-v2/src/common/events.rs @@ -62,6 +62,13 @@ pub enum StorageQueryType { DescendantsHashes, } +impl StorageQueryType { + /// Returns `true` if the query is a descendant query. + pub fn is_descendant_query(&self) -> bool { + matches!(self, Self::DescendantsValues | Self::DescendantsHashes) + } +} + /// The storage result. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] From 0580429905025a9a763821ed26502c95dc8a448d Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 15 Jan 2024 14:03:35 +0200 Subject: [PATCH 19/22] rpc-v2/common: Add `ok` and `err` methods for `ArchiveStorageResult` Signed-off-by: Alexandru Vasile --- .../rpc-spec-v2/src/archive/archive_storage.rs | 15 ++++++--------- substrate/client/rpc-spec-v2/src/common/events.rs | 12 ++++++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs index 351c1aae9a43..408d28dfe84f 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs @@ -76,7 +76,7 @@ where Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, Err(error) => - return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + return ArchiveStorageResult::err(error), } }, StorageQueryType::Hash => @@ -84,14 +84,14 @@ where Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, Err(error) => - return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + return ArchiveStorageResult::err(error), }, StorageQueryType::ClosestDescendantMerkleValue => match self.client.query_merkle_value(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, Err(error) => - return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + return ArchiveStorageResult::err(error), }, StorageQueryType::DescendantsValues => { match self.client.query_iter_pagination( @@ -106,7 +106,7 @@ where ) { Ok((results, _)) => storage_results.extend(results), Err(error) => - return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + return ArchiveStorageResult::err(error), } }, StorageQueryType::DescendantsHashes => { @@ -122,15 +122,12 @@ where ) { Ok((results, _)) => storage_results.extend(results), Err(error) => - return ArchiveStorageResult::Err(ArchiveStorageMethodErr { error }), + return ArchiveStorageResult::err(error), } }, }; } - ArchiveStorageResult::Ok(ArchiveStorageMethodOk { - result: storage_results, - discarded_items, - }) + ArchiveStorageResult::ok(storage_results, discarded_items) } } diff --git a/substrate/client/rpc-spec-v2/src/common/events.rs b/substrate/client/rpc-spec-v2/src/common/events.rs index e30ce4fccc99..b1627d74c844 100644 --- a/substrate/client/rpc-spec-v2/src/common/events.rs +++ b/substrate/client/rpc-spec-v2/src/common/events.rs @@ -113,6 +113,18 @@ pub enum ArchiveStorageResult { Err(ArchiveStorageMethodErr), } +impl ArchiveStorageResult { + /// Create a new `ArchiveStorageResult::Ok` result. + pub fn ok(result: Vec, discarded_items: usize) -> Self { + Self::Ok(ArchiveStorageMethodOk { result, discarded_items }) + } + + /// Create a new `ArchiveStorageResult::Err` result. + pub fn err(error: String) -> Self { + Self::Err(ArchiveStorageMethodErr { error }) + } +} + /// The result of a storage call. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] From 3cf59a8ff73318d18a57efa3a9970916637b59f7 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 15 Jan 2024 14:16:27 +0200 Subject: [PATCH 20/22] rpc-v2/archive: Apply cargo fmt Signed-off-by: Alexandru Vasile --- .../rpc-spec-v2/src/archive/archive_storage.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs index 408d28dfe84f..387f1215862f 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs @@ -75,23 +75,20 @@ where match self.client.query_value(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, - Err(error) => - return ArchiveStorageResult::err(error), + Err(error) => return ArchiveStorageResult::err(error), } }, StorageQueryType::Hash => match self.client.query_hash(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, - Err(error) => - return ArchiveStorageResult::err(error), + Err(error) => return ArchiveStorageResult::err(error), }, StorageQueryType::ClosestDescendantMerkleValue => match self.client.query_merkle_value(hash, &item.key, child_key.as_ref()) { Ok(Some(value)) => storage_results.push(value), Ok(None) => continue, - Err(error) => - return ArchiveStorageResult::err(error), + Err(error) => return ArchiveStorageResult::err(error), }, StorageQueryType::DescendantsValues => { match self.client.query_iter_pagination( @@ -105,8 +102,7 @@ where self.storage_max_reported_items, ) { Ok((results, _)) => storage_results.extend(results), - Err(error) => - return ArchiveStorageResult::err(error), + Err(error) => return ArchiveStorageResult::err(error), } }, StorageQueryType::DescendantsHashes => { @@ -121,8 +117,7 @@ where self.storage_max_reported_items, ) { Ok((results, _)) => storage_results.extend(results), - Err(error) => - return ArchiveStorageResult::err(error), + Err(error) => return ArchiveStorageResult::err(error), } }, }; From 899fa7aa4055714e19a1fb55e2de61dd315af56f Mon Sep 17 00:00:00 2001 From: Alexandru Vasile <60601340+lexnv@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:18:27 +0200 Subject: [PATCH 21/22] Update substrate/client/rpc-spec-v2/src/lib.rs Co-authored-by: Niklas Adolfsson --- substrate/client/rpc-spec-v2/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/rpc-spec-v2/src/lib.rs b/substrate/client/rpc-spec-v2/src/lib.rs index 2381fe4431d4..23ed422cff17 100644 --- a/substrate/client/rpc-spec-v2/src/lib.rs +++ b/substrate/client/rpc-spec-v2/src/lib.rs @@ -78,7 +78,7 @@ pub struct MethodResultErr { pub error: String, } -/// Util function to print the results of rpc-spec-v2 as hex string +/// Util function to encode a value as a hex string pub fn hex_string(data: &Data) -> String { format!("0x{:?}", HexDisplay::from(data)) } From 688e3fbbc0d8d1317f8dbbc513a061921b317e7c Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 15 Jan 2024 14:42:32 +0200 Subject: [PATCH 22/22] archive: Remove unused imports Signed-off-by: Alexandru Vasile --- substrate/client/rpc-spec-v2/src/archive/archive.rs | 2 +- substrate/client/rpc-spec-v2/src/archive/archive_storage.rs | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs index b8c9fc825494..c01afb5d7795 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -20,7 +20,7 @@ use crate::{ archive::{error::Error as ArchiveError, ArchiveApiServer}, - common::events::{ArchiveStorageResult, PaginatedStorageQuery, StorageQueryType}, + common::events::{ArchiveStorageResult, PaginatedStorageQuery}, hex_string, MethodResult, }; diff --git a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs index 387f1215862f..09415af1ca13 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive_storage.rs @@ -24,10 +24,7 @@ use sc_client_api::{Backend, ChildInfo, StorageKey, StorageProvider}; use sp_runtime::traits::Block as BlockT; use crate::common::{ - events::{ - ArchiveStorageMethodErr, ArchiveStorageMethodOk, ArchiveStorageResult, - PaginatedStorageQuery, StorageQueryType, - }, + events::{ArchiveStorageResult, PaginatedStorageQuery, StorageQueryType}, storage::{IterQueryType, QueryIter, Storage}, };