From 1ceababcae230ded04a04e08636a9bec98a7cc83 Mon Sep 17 00:00:00 2001 From: BGluth Date: Mon, 10 Jun 2024 10:29:35 -0600 Subject: [PATCH 1/5] Set zk_evm deps to use crates.io --- Cargo.lock | 52 +++++++++++++++++++++++++++++++++++------------ Cargo.toml | 4 ++-- common/src/lib.rs | 2 +- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ecbf55f..57694fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -517,8 +517,9 @@ dependencies = [ [[package]] name = "evm_arithmetization" -version = "0.1.1" -source = "git+https://github.com/0xPolygonZero/zk_evm.git?branch=robins_not_so_awesome_branch_for_brendan#ab7c6cbda46e77877a8462af33bb5a51c714ca97" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6abdbb2da0df19b2d734ce9bc5edd1e12e383e3dc999075c9579a12aad2d3c" dependencies = [ "anyhow", "bytes", @@ -708,6 +709,17 @@ dependencies = [ "parity-scale-codec", ] +[[package]] +name = "impl-num-traits" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "951641f13f873bff03d4bf19ae8bec531935ac0ac2cc775f84d7edfdcfed3f17" +dependencies = [ + "integer-sqrt", + "num-traits", + "uint", +] + [[package]] name = "impl-rlp" version = "0.3.0" @@ -759,6 +771,15 @@ dependencies = [ "serde", ] +[[package]] +name = "integer-sqrt" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "276ec31bcb4a9ee45f58bec6f9ec700ae4cf4f4f8f2fa7e06cb406bd5ffdd770" +dependencies = [ + "num-traits", +] + [[package]] name = "is-terminal" version = "0.4.12" @@ -874,13 +895,18 @@ dependencies = [ [[package]] name = "mpt_trie" -version = "0.1.1" -source = "git+https://github.com/0xPolygonZero/zk_evm.git?branch=robins_not_so_awesome_branch_for_brendan#ab7c6cbda46e77877a8462af33bb5a51c714ca97" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d1369308dd83c0c5b09ef9fc966846b76b7ee1548f5a07348a52ac8a209cf1f" dependencies = [ "bytes", "enum-as-inner", "ethereum-types", "hex", + "impl-codec", + "impl-num-traits", + "impl-rlp", + "impl-serde", "keccak-hash 0.10.0", "log", "num", @@ -1086,9 +1112,9 @@ dependencies = [ [[package]] name = "plonky2" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b25deb9a4b9c2014c2f99cd36078f30e453d188d0ca8dd4c5ffd1d494b661ac1" +checksum = "85f26b090b989aebdeaf6a4eed748c1fbcabf67e7273a22e4e0c877b63846d0f" dependencies = [ "ahash", "anyhow", @@ -1111,9 +1137,9 @@ dependencies = [ [[package]] name = "plonky2_field" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465a741ba134485af571152aab5086457a470aa8893391186cf78dd389694440" +checksum = "3a1dca60ad900d81b1fe2df3d0b88d43345988e2935e6709176e96573f4bcf5d" dependencies = [ "anyhow", "itertools", @@ -1432,9 +1458,9 @@ checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "starky" -version = "0.2.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e0a1eec739c7a67cb1c6f916c0b7bf2d281cf2edb35d3db5caa6989090133e" +checksum = "8a8480ca5b8eedf83ad070a780783b4e21a56c6ef66b4c0d1b7520b72bdfda1b" dependencies = [ "ahash", "anyhow", @@ -1581,8 +1607,9 @@ dependencies = [ [[package]] name = "trace_decoder" -version = "0.1.1" -source = "git+https://github.com/0xPolygonZero/zk_evm.git?branch=robins_not_so_awesome_branch_for_brendan#ab7c6cbda46e77877a8462af33bb5a51c714ca97" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5b92593c1a3417d9bf81e647a827874bc55a4c04683c742ebc12a460fd2a8f" dependencies = [ "bytes", "ciborium", @@ -1599,7 +1626,6 @@ dependencies = [ "rlp", "rlp-derive", "serde", - "serde_json", "serde_with", "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index 2e7b792..f0e226f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.dependencies] anyhow = { version = "1.0.75", features = ["backtrace"] } clap = { version = "4.4.6", features = ["derive", "env"] } -mpt_trie = { version = "0.1.0", git = "https://github.com/0xPolygonZero/zk_evm.git", branch = "robins_not_so_awesome_branch_for_brendan" } +mpt_trie = "0.2.1" pretty_env_logger = "0.5.0" -trace_decoder = { version = "0.1.0", git = "https://github.com/0xPolygonZero/zk_evm.git", branch = "robins_not_so_awesome_branch_for_brendan" } +trace_decoder = "0.3.1" serde_json = "1.0.114" diff --git a/common/src/lib.rs b/common/src/lib.rs index 6a27051..1960157 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -14,7 +14,7 @@ pub fn read_input_from_file(p: &Path) -> HashedPartialTrie { { "compact" => { let out = process_compact_prestate_debug(read_compact_from_file(p)).unwrap(); - out.witness_out.tries.state + out.witness_out.state_trie } "json" => read_json_trie_from_file(p), _ => panic!( From 3cf3ff8073afab63c4f583b3ba381b1a47f26f1b Mon Sep 17 00:00:00 2001 From: BGluth Date: Mon, 10 Jun 2024 12:43:55 -0600 Subject: [PATCH 2/5] Comments, query config, and trie_stats incorporation - Added some comments on how to use the tool, specifically regarding what format the trie files should be in. --- Cargo.lock | 12 +++++++++++ Cargo.toml | 2 +- common/Cargo.toml | 1 + common/src/lib.rs | 7 +++++++ trie_diff/src/main.rs | 6 +++++- trie_query/src/main.rs | 7 ++++++- trie_stats/Cargo.toml | 14 +++++++++++++ trie_stats/src/main.rs | 47 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 trie_stats/Cargo.toml create mode 100644 trie_stats/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 57694fc..26930a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -303,6 +303,7 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" name = "common" version = "0.1.0" dependencies = [ + "clap", "hex", "mpt_trie", "serde_json", @@ -1656,6 +1657,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "trie_stats" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "common", + "mpt_trie", + "pretty_env_logger", +] + [[package]] name = "typenum" version = "1.17.0" diff --git a/Cargo.toml b/Cargo.toml index f0e226f..5d1cfb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [ "common", "trie_diff", "trie_query"] +members = [ "common", "trie_diff", "trie_query", "trie_stats" ] resolver = "2" [workspace.dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 4262f49..5fcc2e0 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +clap = { workspace = true } mpt_trie = { workspace = true } hex = "0.4.3" diff --git a/common/src/lib.rs b/common/src/lib.rs index 1960157..4685958 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -6,12 +6,19 @@ use trace_decoder::{ trace_protocol::TrieCompact, }; +pub const TRIE_PATH_DOC_STR: &str = r#" +Each path must point to a file that is either: +- (`*.compact`) Compact encoding (hex string) of an MPT trie (spec: https://gist.github.com/mandrigin/ff7eccf30d0ef9c572bafcb0ab665cff). +- (`*.json`) A serialized `HashedPartialTrie`. +"#; + pub fn read_input_from_file(p: &Path) -> HashedPartialTrie { match p .extension() .map(|s| s.to_str().unwrap_or_default()) .unwrap_or_default() { + // For now, if we ever get compact, we're going to just use the state trie. "compact" => { let out = process_compact_prestate_debug(read_compact_from_file(p)).unwrap(); out.witness_out.state_trie diff --git a/trie_diff/src/main.rs b/trie_diff/src/main.rs index 23ddffa..d791309 100644 --- a/trie_diff/src/main.rs +++ b/trie_diff/src/main.rs @@ -1,11 +1,15 @@ use std::path::PathBuf; use clap::Parser; -use common::read_input_from_file; +use common::{read_input_from_file, TRIE_PATH_DOC_STR}; use mpt_trie::debug_tools::diff::create_diff_between_tries; use mpt_trie::partial_trie::PartialTrie; #[derive(Debug, Parser)] +#[command(after_help = TRIE_PATH_DOC_STR)] +/// Attempts to find the lowest possible point where two given tries differ. +/// +/// Takes in two paths to tries that we want to include in the diff. struct ProgArgs { a_path: PathBuf, b_path: PathBuf, diff --git a/trie_query/src/main.rs b/trie_query/src/main.rs index 5e2bd86..a309e87 100644 --- a/trie_query/src/main.rs +++ b/trie_query/src/main.rs @@ -1,15 +1,20 @@ use std::path::PathBuf; use clap::Parser; -use common::read_input_from_file; +use common::{read_input_from_file, TRIE_PATH_DOC_STR}; use mpt_trie::{ debug_tools::query::{get_path_from_query, DebugQueryParamsBuilder}, nibbles::Nibbles, }; +/// Displays a "trace" of a key query for a given key, where the trace contains the nodes encountered. #[derive(Debug, Parser)] +#[command(after_help = TRIE_PATH_DOC_STR)] struct ProgArgs { + /// Path to the trie to query against. trie_path: PathBuf, + + /// The key to query in hex format (with or without the "0x"). key: Nibbles, } diff --git a/trie_stats/Cargo.toml b/trie_stats/Cargo.toml new file mode 100644 index 0000000..a2b4767 --- /dev/null +++ b/trie_stats/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "trie_stats" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +common = { path = "../common" } + +anyhow = { workspace = true } +clap = { workspace = true } +mpt_trie ={ workspace = true } +pretty_env_logger = { workspace = true } diff --git a/trie_stats/src/main.rs b/trie_stats/src/main.rs new file mode 100644 index 0000000..65ea11b --- /dev/null +++ b/trie_stats/src/main.rs @@ -0,0 +1,47 @@ +use std::path::PathBuf; + +use clap::Parser; +use common::{read_input_from_file, TRIE_PATH_DOC_STR}; +use mpt_trie::debug_tools::stats::get_trie_stats_with_name; + +/// Analyze one or two tries and output stats. +#[derive(Debug, Parser)] +#[command(after_help = TRIE_PATH_DOC_STR)] +struct ProgArgs { + /// The primary trie to compare against. + trie_path: PathBuf, + + /// If a second trie is provided, a comparison will be made between the two tries along with printout out their individual stats. + other_trie_path: Option, +} + +fn main() { + pretty_env_logger::init(); + + let p_args = ProgArgs::parse(); + let trie = read_input_from_file(&p_args.trie_path); + let trie_name = p_args + .trie_path + .file_name() + .map(|os_str| os_str.to_string_lossy()) + .unwrap_or("Trie".into()); + + let stats = get_trie_stats_with_name(&trie, trie_name.to_string()); + println!("{}", stats); + + if let Some(other_trie_path) = p_args.other_trie_path { + // A second trie was passed in. Print its stats and also do a comparison. + let other_trie = read_input_from_file(&other_trie_path); + let other_trie_name = other_trie_path + .file_name() + .map(|os_str| os_str.to_string_lossy()) + .unwrap_or("Trie".into()); + let other_trie_stats = get_trie_stats_with_name(&other_trie, other_trie_name.into()); + + println!("{}", other_trie_stats); + + let comparison = stats.compare(&other_trie_stats); + + println!("{}", comparison); + } +} From cd73902d6a9d134958235f75c2f0f495d299ebe8 Mon Sep 17 00:00:00 2001 From: BGluth Date: Thu, 13 Jun 2024 12:21:19 -0600 Subject: [PATCH 3/5] Added changes that somehow did not get in the last commit --- trie_query/src/main.rs | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/trie_query/src/main.rs b/trie_query/src/main.rs index a309e87..bb1d592 100644 --- a/trie_query/src/main.rs +++ b/trie_query/src/main.rs @@ -7,6 +7,8 @@ use mpt_trie::{ nibbles::Nibbles, }; +const QUERY_CONFIG_STR: &str = "Query Config"; + /// Displays a "trace" of a key query for a given key, where the trace contains the nodes encountered. #[derive(Debug, Parser)] #[command(after_help = TRIE_PATH_DOC_STR)] @@ -16,6 +18,34 @@ struct ProgArgs { /// The key to query in hex format (with or without the "0x"). key: Nibbles, + + /// Config for the query output. + #[command(flatten)] + config: QueryConfig, +} + +#[derive(Parser, Debug)] +struct QueryConfig { + /// Include the key piece (if the node contains a piece of the key) contained in each node. + #[arg(short = 'k', long, default_value_t = true, help_heading = QUERY_CONFIG_STR)] + include_key_piece_per_node: bool, + + /// Include the node types in the trace. + #[arg(short = 't', long, default_value_t = true, help_heading = QUERY_CONFIG_STR)] + include_node_type: bool, + + /// Include additional info that is specific to a given node type. + #[arg(short = 's', long, default_value_t = false, help_heading = QUERY_CONFIG_STR)] + include_node_specific_values: bool, +} + +impl From for DebugQueryParamsBuilder { + fn from(v: QueryConfig) -> Self { + DebugQueryParamsBuilder::default() + .print_key_pieces(v.include_key_piece_per_node) + .print_node_type(v.include_node_type) + .print_node_specific_values(v.include_node_specific_values) + } } fn main() -> anyhow::Result<()> { @@ -24,9 +54,7 @@ fn main() -> anyhow::Result<()> { let p_args = ProgArgs::parse(); let trie = read_input_from_file(&p_args.trie_path); - let query = DebugQueryParamsBuilder::default() - .print_node_specific_values(true) - .build(p_args.key); + let query = DebugQueryParamsBuilder::from(p_args.config).build(p_args.key); let res = get_path_from_query(&trie, query); println!("{}", res); From 0946575901ff6e5e95240abadddeaa504c35e15a Mon Sep 17 00:00:00 2001 From: BGluth Date: Thu, 13 Jun 2024 12:21:56 -0600 Subject: [PATCH 4/5] Bumped deps (PR request) --- Cargo.lock | 12 ++++++------ Cargo.toml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26930a4..1ed5eb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,9 +518,9 @@ dependencies = [ [[package]] name = "evm_arithmetization" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6abdbb2da0df19b2d734ce9bc5edd1e12e383e3dc999075c9579a12aad2d3c" +checksum = "87d958d9badfbbd52c003da99378ff08c11f16efd4c4729e0fe2ca2224c46e8e" dependencies = [ "anyhow", "bytes", @@ -896,9 +896,9 @@ dependencies = [ [[package]] name = "mpt_trie" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d1369308dd83c0c5b09ef9fc966846b76b7ee1548f5a07348a52ac8a209cf1f" +checksum = "c3febfed5b65d35c9b9811733e811a48b0591ed16dc07bee08c213c888b974b8" dependencies = [ "bytes", "enum-as-inner", @@ -1608,9 +1608,9 @@ dependencies = [ [[package]] name = "trace_decoder" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5b92593c1a3417d9bf81e647a827874bc55a4c04683c742ebc12a460fd2a8f" +checksum = "6b86ec3b0b2f16b755f7f4acff4faa48f0a5458c83cbc8b0ec64557ded12c0f2" dependencies = [ "bytes", "ciborium", diff --git a/Cargo.toml b/Cargo.toml index 5d1cfb9..f781263 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.dependencies] anyhow = { version = "1.0.75", features = ["backtrace"] } clap = { version = "4.4.6", features = ["derive", "env"] } -mpt_trie = "0.2.1" +mpt_trie = "0.3.0" pretty_env_logger = "0.5.0" -trace_decoder = "0.3.1" +trace_decoder = "0.4.0" serde_json = "1.0.114" From c52ec9904a9abc972a22cd8dd6ca79c4fac30ea4 Mon Sep 17 00:00:00 2001 From: Robin Salen <30937548+Nashtare@users.noreply.github.com> Date: Fri, 14 Jun 2024 07:21:44 +0900 Subject: [PATCH 5/5] Update trie_query/src/main.rs --- trie_query/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie_query/src/main.rs b/trie_query/src/main.rs index bb1d592..b162178 100644 --- a/trie_query/src/main.rs +++ b/trie_query/src/main.rs @@ -26,7 +26,7 @@ struct ProgArgs { #[derive(Parser, Debug)] struct QueryConfig { - /// Include the key piece (if the node contains a piece of the key) contained in each node. + /// Include the key piece (if the node contains a piece of the key) for each node. #[arg(short = 'k', long, default_value_t = true, help_heading = QUERY_CONFIG_STR)] include_key_piece_per_node: bool,