From 1cff666b5206c28e3116eea7edb36b65135775a1 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Wed, 4 Sep 2024 11:26:26 -0400 Subject: [PATCH] Create a Basic Proving Trie for the Runtime (#3881) This PR will introduce a `BasicProvingTrie` type, which makes it easy to construct and prove data in a base-16 merkle trie within the runtime. Data into the merkle trie only require that they implement `Encode` / `Decode`. A FRAME compatible `TrieError` was created and added to `DispatchError`. Expected usage is to construct the merkle trie with all data offline, and then place only the merkle root of that trie on-chain. Also offchain, a user is given a compact merkle proof of some data they want to prove exists on the blockchain. Then in the runtime, you can call `verify_single_value_proof` or `verify_proof` with the root, proof, and the keys and values you want to verify exists in the merkle trie. Closes https://github.com/paritytech/polkadot-sdk/issues/3880 Contributes to #5400 --------- Co-authored-by: Ankan <10196091+Ank4n@users.noreply.github.com> Co-authored-by: Oliver Tale-Yazdi --- Cargo.lock | 1 + prdoc/pr_3881.prdoc | 15 + substrate/primitives/runtime/Cargo.toml | 2 + substrate/primitives/runtime/src/lib.rs | 16 + .../primitives/runtime/src/proving_trie.rs | 388 ++++++++++++++++++ 5 files changed, 422 insertions(+) create mode 100644 prdoc/pr_3881.prdoc create mode 100644 substrate/primitives/runtime/src/proving_trie.rs diff --git a/Cargo.lock b/Cargo.lock index b3eefe1826c9..508fc2e89221 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20741,6 +20741,7 @@ dependencies = [ "sp-state-machine", "sp-std 14.0.0", "sp-tracing 16.0.0", + "sp-trie", "sp-weights", "substrate-test-runtime-client", "tracing", diff --git a/prdoc/pr_3881.prdoc b/prdoc/pr_3881.prdoc new file mode 100644 index 000000000000..4cf6425e73a5 --- /dev/null +++ b/prdoc/pr_3881.prdoc @@ -0,0 +1,15 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Introduce a Generic Proving Trie + +doc: + - audience: Runtime Dev + description: | + This PR introduces a Proving Trie object which can be used inside the runtime. This can allow + for things like airdrops where a single hash is stored on chain representing the whole airdrop + and individuals present a proof of their inclusion in the airdrop. + +crates: + - name: sp-runtime + bump: major diff --git a/substrate/primitives/runtime/Cargo.toml b/substrate/primitives/runtime/Cargo.toml index 55e9f4b31995..800bf4bd0737 100644 --- a/substrate/primitives/runtime/Cargo.toml +++ b/substrate/primitives/runtime/Cargo.toml @@ -32,6 +32,7 @@ sp-arithmetic = { workspace = true } sp-core = { workspace = true } sp-io = { workspace = true } sp-std = { workspace = true } +sp-trie = { workspace = true } sp-weights = { workspace = true } docify = { workspace = true } tracing = { workspace = true, features = ["log"], default-features = false } @@ -69,6 +70,7 @@ std = [ "sp-state-machine/std", "sp-std/std", "sp-tracing/std", + "sp-trie/std", "sp-weights/std", "tracing/std", ] diff --git a/substrate/primitives/runtime/src/lib.rs b/substrate/primitives/runtime/src/lib.rs index fd10dee2a7c5..ba1ea3769724 100644 --- a/substrate/primitives/runtime/src/lib.rs +++ b/substrate/primitives/runtime/src/lib.rs @@ -90,6 +90,7 @@ pub mod generic; pub mod legacy; mod multiaddress; pub mod offchain; +pub mod proving_trie; pub mod runtime_logger; mod runtime_string; #[cfg(feature = "std")] @@ -103,6 +104,8 @@ pub use crate::runtime_string::*; // Re-export Multiaddress pub use multiaddress::MultiAddress; +use proving_trie::TrieError; + /// Re-export these since they're only "kind of" generic. pub use generic::{Digest, DigestItem}; @@ -592,6 +595,8 @@ pub enum DispatchError { Unavailable, /// Root origin is not allowed. RootNotAllowed, + /// An error with tries. + Trie(TrieError), } /// Result of a `Dispatchable` which contains the `DispatchResult` and additional information about @@ -697,6 +702,12 @@ impl From for DispatchError { } } +impl From for DispatchError { + fn from(e: TrieError) -> DispatchError { + Self::Trie(e) + } +} + impl From<&'static str> for DispatchError { fn from(err: &'static str) -> DispatchError { Self::Other(err) @@ -721,6 +732,7 @@ impl From for &'static str { Corruption => "State corrupt", Unavailable => "Resource unavailable", RootNotAllowed => "Root not allowed", + Trie(e) => e.into(), } } } @@ -768,6 +780,10 @@ impl traits::Printable for DispatchError { Corruption => "State corrupt".print(), Unavailable => "Resource unavailable".print(), RootNotAllowed => "Root not allowed".print(), + Trie(e) => { + "Trie error: ".print(); + <&'static str>::from(*e).print(); + }, } } } diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs new file mode 100644 index 000000000000..688bf81e0d7b --- /dev/null +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -0,0 +1,388 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Types for a compact base-16 merkle trie used for checking and generating proofs within the +//! runtime. The `sp-trie` crate exposes all of these same functionality (and more), but this +//! library is designed to work more easily with runtime native types, which simply need to +//! implement `Encode`/`Decode`. It also exposes a runtime friendly `TrieError` type which can be +//! use inside of a FRAME Pallet. +//! +//! Proofs are created with latest substrate trie format (`LayoutV1`), and are not compatible with +//! proofs using `LayoutV0`. + +use crate::{Decode, DispatchError, Encode, MaxEncodedLen, TypeInfo}; +#[cfg(feature = "serde")] +use crate::{Deserialize, Serialize}; + +use sp_std::vec::Vec; +use sp_trie::{ + trie_types::{TrieDBBuilder, TrieDBMutBuilderV1, TrieError as SpTrieError}, + LayoutV1, MemoryDB, Trie, TrieMut, VerifyError, +}; + +type HashOf = ::Out; + +/// A runtime friendly error type for tries. +#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, Debug, TypeInfo, MaxEncodedLen)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub enum TrieError { + /* From TrieError */ + /// Attempted to create a trie with a state root not in the DB. + InvalidStateRoot, + /// Trie item not found in the database, + IncompleteDatabase, + /// A value was found in the trie with a nibble key that was not byte-aligned. + ValueAtIncompleteKey, + /// Corrupt Trie item. + DecoderError, + /// Hash is not value. + InvalidHash, + /* From VerifyError */ + /// The statement being verified contains multiple key-value pairs with the same key. + DuplicateKey, + /// The proof contains at least one extraneous node. + ExtraneousNode, + /// The proof contains at least one extraneous value which should have been omitted from the + /// proof. + ExtraneousValue, + /// The proof contains at least one extraneous hash reference the should have been omitted. + ExtraneousHashReference, + /// The proof contains an invalid child reference that exceeds the hash length. + InvalidChildReference, + /// The proof indicates that an expected value was not found in the trie. + ValueMismatch, + /// The proof is missing trie nodes required to verify. + IncompleteProof, + /// The root hash computed from the proof is incorrect. + RootMismatch, + /// One of the proof nodes could not be decoded. + DecodeError, +} + +impl From> for TrieError { + fn from(error: SpTrieError) -> Self { + match error { + SpTrieError::InvalidStateRoot(..) => Self::InvalidStateRoot, + SpTrieError::IncompleteDatabase(..) => Self::IncompleteDatabase, + SpTrieError::ValueAtIncompleteKey(..) => Self::ValueAtIncompleteKey, + SpTrieError::DecoderError(..) => Self::DecoderError, + SpTrieError::InvalidHash(..) => Self::InvalidHash, + } + } +} + +impl From> for TrieError { + fn from(error: VerifyError) -> Self { + match error { + VerifyError::DuplicateKey(..) => Self::DuplicateKey, + VerifyError::ExtraneousNode => Self::ExtraneousNode, + VerifyError::ExtraneousValue(..) => Self::ExtraneousValue, + VerifyError::ExtraneousHashReference(..) => Self::ExtraneousHashReference, + VerifyError::InvalidChildReference(..) => Self::InvalidChildReference, + VerifyError::ValueMismatch(..) => Self::ValueMismatch, + VerifyError::IncompleteProof => Self::IncompleteProof, + VerifyError::RootMismatch(..) => Self::RootMismatch, + VerifyError::DecodeError(..) => Self::DecodeError, + } + } +} + +impl From for &'static str { + fn from(e: TrieError) -> &'static str { + match e { + TrieError::InvalidStateRoot => "The state root is not in the database.", + TrieError::IncompleteDatabase => "A trie item was not found in the database.", + TrieError::ValueAtIncompleteKey => + "A value was found with a key that is not byte-aligned.", + TrieError::DecoderError => "A corrupt trie item was encountered.", + TrieError::InvalidHash => "The hash does not match the expected value.", + TrieError::DuplicateKey => "The proof contains duplicate keys.", + TrieError::ExtraneousNode => "The proof contains extraneous nodes.", + TrieError::ExtraneousValue => "The proof contains extraneous values.", + TrieError::ExtraneousHashReference => "The proof contains extraneous hash references.", + TrieError::InvalidChildReference => "The proof contains an invalid child reference.", + TrieError::ValueMismatch => "The proof indicates a value mismatch.", + TrieError::IncompleteProof => "The proof is incomplete.", + TrieError::RootMismatch => "The root hash computed from the proof is incorrect.", + TrieError::DecodeError => "One of the proof nodes could not be decoded.", + } + } +} + +/// A helper structure for building a basic base-16 merkle trie and creating compact proofs for that +/// trie. Proofs are created with latest substrate trie format (`LayoutV1`), and are not compatible +/// with proofs using `LayoutV0`. +pub struct BasicProvingTrie +where + Hashing: sp_core::Hasher, +{ + db: MemoryDB, + root: HashOf, + _phantom: core::marker::PhantomData<(Key, Value)>, +} + +impl BasicProvingTrie +where + Hashing: sp_core::Hasher, + Key: Encode, + Value: Encode + Decode, +{ + /// Create a new instance of a `ProvingTrie` using an iterator of key/value pairs. + pub fn generate_for(items: I) -> Result + where + I: IntoIterator, + { + let mut db = MemoryDB::default(); + let mut root = Default::default(); + + { + let mut trie = TrieDBMutBuilderV1::new(&mut db, &mut root).build(); + for (key, value) in items.into_iter() { + key.using_encoded(|k| value.using_encoded(|v| trie.insert(k, v))) + .map_err(|_| "failed to insert into trie")?; + } + } + + Ok(Self { db, root, _phantom: Default::default() }) + } + + /// Access the underlying trie root. + pub fn root(&self) -> &HashOf { + &self.root + } + + /// Query a value contained within the current trie. Returns `None` if the + /// nodes within the current `MemoryDB` are insufficient to query the item. + pub fn query(&self, key: Key) -> Option { + let trie = TrieDBBuilder::new(&self.db, &self.root).build(); + key.using_encoded(|s| trie.get(s)) + .ok()? + .and_then(|raw| Value::decode(&mut &*raw).ok()) + } + + /// Create a compact merkle proof needed to prove all `keys` and their values are in the trie. + /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a + /// proof. + /// + /// This function makes a proof with latest substrate trie format (`LayoutV1`), and is not + /// compatible with `LayoutV0`. + /// + /// When verifying the proof created by this function, you must include all of the keys and + /// values of the proof, else the verifier will complain that extra nodes are provided in the + /// proof that are not needed. + pub fn create_proof(&self, keys: &[Key]) -> Result>, DispatchError> { + sp_trie::generate_trie_proof::, _, _, _>( + &self.db, + self.root, + &keys.into_iter().map(|k| k.encode()).collect::>>(), + ) + .map_err(|err| TrieError::from(*err).into()) + } + + /// Create a compact merkle proof needed to prove a single key and its value are in the trie. + /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a + /// proof. + /// + /// This function makes a proof with latest substrate trie format (`LayoutV1`), and is not + /// compatible with `LayoutV0`. + pub fn create_single_value_proof(&self, key: Key) -> Result>, DispatchError> { + self.create_proof(&[key]) + } +} + +/// Verify the existence or non-existence of `key` and `value` in a given trie root and proof. +/// +/// Proofs must be created with latest substrate trie format (`LayoutV1`). +pub fn verify_single_value_proof( + root: HashOf, + proof: &[Vec], + key: Key, + maybe_value: Option, +) -> Result<(), DispatchError> +where + Hashing: sp_core::Hasher, + Key: Encode, + Value: Encode, +{ + sp_trie::verify_trie_proof::, _, _, _>( + &root, + proof, + &[(key.encode(), maybe_value.map(|value| value.encode()))], + ) + .map_err(|err| TrieError::from(err).into()) +} + +/// Verify the existence or non-existence of multiple `items` in a given trie root and proof. +/// +/// Proofs must be created with latest substrate trie format (`LayoutV1`). +pub fn verify_proof( + root: HashOf, + proof: &[Vec], + items: &[(Key, Option)], +) -> Result<(), DispatchError> +where + Hashing: sp_core::Hasher, + Key: Encode, + Value: Encode, +{ + let items_encoded = items + .into_iter() + .map(|(key, maybe_value)| (key.encode(), maybe_value.as_ref().map(|value| value.encode()))) + .collect::, Option>)>>(); + + sp_trie::verify_trie_proof::, _, _, _>(&root, proof, &items_encoded) + .map_err(|err| TrieError::from(err).into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::BlakeTwo256; + use sp_core::H256; + use sp_std::collections::btree_map::BTreeMap; + + // A trie which simulates a trie of accounts (u32) and balances (u128). + type BalanceTrie = BasicProvingTrie; + + // The expected root hash for an empty trie. + fn empty_root() -> H256 { + sp_trie::empty_trie_root::>() + } + + fn create_balance_trie() -> BalanceTrie { + // Create a map of users and their balances. + let mut map = BTreeMap::::new(); + for i in 0..100u32 { + map.insert(i, i.into()); + } + + // Put items into the trie. + let balance_trie = BalanceTrie::generate_for(map).unwrap(); + + // Root is changed. + let root = *balance_trie.root(); + assert!(root != empty_root()); + + // Assert valid keys are queryable. + assert_eq!(balance_trie.query(6u32), Some(6u128)); + assert_eq!(balance_trie.query(9u32), Some(9u128)); + assert_eq!(balance_trie.query(69u32), Some(69u128)); + // Invalid key returns none. + assert_eq!(balance_trie.query(6969u32), None); + + balance_trie + } + + #[test] + fn empty_trie_works() { + let empty_trie = BalanceTrie::generate_for(Vec::new()).unwrap(); + assert_eq!(*empty_trie.root(), empty_root()); + } + + #[test] + fn basic_end_to_end_single_value() { + let balance_trie = create_balance_trie(); + let root = *balance_trie.root(); + + // Create a proof for a valid key. + let proof = balance_trie.create_single_value_proof(6u32).unwrap(); + + // Assert key is provable, all other keys are invalid. + for i in 0..200u32 { + if i == 6 { + assert_eq!( + verify_single_value_proof::( + root, + &proof, + i, + Some(u128::from(i)) + ), + Ok(()) + ); + // Wrong value is invalid. + assert_eq!( + verify_single_value_proof::( + root, + &proof, + i, + Some(u128::from(i + 1)) + ), + Err(TrieError::RootMismatch.into()) + ); + } else { + assert!(verify_single_value_proof::( + root, + &proof, + i, + Some(u128::from(i)) + ) + .is_err()); + assert!(verify_single_value_proof::( + root, + &proof, + i, + None:: + ) + .is_err()); + } + } + } + + #[test] + fn basic_end_to_end_multi_value() { + let balance_trie = create_balance_trie(); + let root = *balance_trie.root(); + + // Create a proof for a valid and invalid key. + let proof = balance_trie.create_proof(&[6u32, 69u32, 6969u32]).unwrap(); + let items = [(6u32, Some(6u128)), (69u32, Some(69u128)), (6969u32, None)]; + + assert_eq!(verify_proof::(root, &proof, &items), Ok(())); + } + + #[test] + fn proof_fails_with_bad_data() { + let balance_trie = create_balance_trie(); + let root = *balance_trie.root(); + + // Create a proof for a valid key. + let proof = balance_trie.create_single_value_proof(6u32).unwrap(); + + // Correct data verifies successfully + assert_eq!( + verify_single_value_proof::(root, &proof, 6u32, Some(6u128)), + Ok(()) + ); + + // Fail to verify proof with wrong root + assert_eq!( + verify_single_value_proof::( + Default::default(), + &proof, + 6u32, + Some(6u128) + ), + Err(TrieError::RootMismatch.into()) + ); + + // Fail to verify proof with wrong data + assert_eq!( + verify_single_value_proof::(root, &[], 6u32, Some(6u128)), + Err(TrieError::IncompleteProof.into()) + ); + } +}