diff --git a/aptos-move/e2e-benchmark/src/main.rs b/aptos-move/e2e-benchmark/src/main.rs index 7169989088d135..6ac46ffd3f7856 100644 --- a/aptos-move/e2e-benchmark/src/main.rs +++ b/aptos-move/e2e-benchmark/src/main.rs @@ -15,7 +15,7 @@ use aptos_transaction_generator_lib::{ use aptos_types::{account_address::AccountAddress, transaction::TransactionPayload}; use rand::{rngs::StdRng, SeedableRng}; use serde_json::json; -use std::process::exit; +use std::{collections::HashMap, process::exit}; pub fn execute_txn( executor: &mut FakeExecutor, @@ -80,10 +80,55 @@ const ALLOWED_REGRESSION: f32 = 0.15; const ALLOWED_IMPROVEMENT: f32 = 0.15; const ABSOLUTE_BUFFER_US: f32 = 2.0; +const CALIBRATION_VALUES: &str = " +Loop { loop_count: Some(100000), loop_type: NoOp } 6 0.988 1.039 41212.4 +Loop { loop_count: Some(10000), loop_type: Arithmetic } 6 0.977 1.038 25868.8 +CreateObjects { num_objects: 10, object_payload_size: 0 } 6 0.940 1.026 152.1 +CreateObjects { num_objects: 10, object_payload_size: 10240 } 6 0.934 1.051 9731.3 +CreateObjects { num_objects: 100, object_payload_size: 0 } 6 0.966 1.051 1458.3 +CreateObjects { num_objects: 100, object_payload_size: 10240 } 6 0.969 1.077 11196.4 +InitializeVectorPicture { length: 40 } 6 0.973 1.066 75.0 +VectorPicture { length: 40 } 6 0.955 1.092 22.0 +VectorPictureRead { length: 40 } 6 0.952 1.047 21.0 +InitializeVectorPicture { length: 30720 } 6 0.969 1.071 27295.8 +VectorPicture { length: 30720 } 6 0.957 1.066 6560.2 +VectorPictureRead { length: 30720 } 6 0.948 1.053 6642.8 +SmartTablePicture { length: 30720, num_points_per_txn: 200 } 6 0.972 1.024 42660.4 +SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 6 0.961 1.020 73725.5 +ResourceGroupsSenderWriteTag { string_length: 1024 } 6 0.867 1.001 15.0 +ResourceGroupsSenderMultiChange { string_length: 1024 } 6 0.966 1.069 29.0 +TokenV1MintAndTransferFT 6 0.972 1.045 356.8 +TokenV1MintAndTransferNFTSequential 6 0.991 1.067 543.7 +TokenV2AmbassadorMint { numbered: true } 6 0.987 1.052 474.4 +LiquidityPoolSwap { is_stable: true } 6 0.970 1.042 555.4 +LiquidityPoolSwap { is_stable: false } 6 0.925 1.001 535.3 +"; + +struct CalibrationInfo { + // count: usize, + expected_time: f32, +} + +fn get_parsed_calibration_values() -> HashMap { + CALIBRATION_VALUES + .trim() + .split('\n') + .map(|line| { + let parts = line.split('\t').collect::>(); + (parts[0].to_string(), CalibrationInfo { + // count: parts[1].parse().unwrap(), + expected_time: parts[parts.len() - 1].parse().unwrap(), + }) + }) + .collect() +} + fn main() { let executor = FakeExecutor::from_head_genesis(); let mut executor = executor.set_not_parallel(); + let calibration_values = get_parsed_calibration_values(); + let entry_points = vec![ // too fast for the timer // (, EntryPoints::Nop), @@ -91,59 +136,57 @@ fn main() { // data_length: Some(32), // }), // (, EntryPoints::IncGlobal), - (34651, EntryPoints::Loop { + EntryPoints::Loop { loop_count: Some(100000), loop_type: LoopType::NoOp, - }), - (21145, EntryPoints::Loop { + }, + EntryPoints::Loop { loop_count: Some(10000), loop_type: LoopType::Arithmetic, - }), + }, // This is a cheap bcs (serializing vec), so not representative of what BCS native call should cost. // (, EntryPoints::Loop { loop_count: Some(1000), loop_type: LoopType::BCS { len: 1024 }}), - (124, EntryPoints::CreateObjects { + EntryPoints::CreateObjects { num_objects: 10, object_payload_size: 0, - }), - (8090, EntryPoints::CreateObjects { + }, + EntryPoints::CreateObjects { num_objects: 10, object_payload_size: 10 * 1024, - }), - (1246, EntryPoints::CreateObjects { + }, + EntryPoints::CreateObjects { num_objects: 100, object_payload_size: 0, - }), - (9556, EntryPoints::CreateObjects { + }, + EntryPoints::CreateObjects { num_objects: 100, object_payload_size: 10 * 1024, - }), - (61, EntryPoints::InitializeVectorPicture { length: 40 }), - (16, EntryPoints::VectorPicture { length: 40 }), - (16, EntryPoints::VectorPictureRead { length: 40 }), - (23256, EntryPoints::InitializeVectorPicture { - length: 30 * 1024, - }), - (5860, EntryPoints::VectorPicture { length: 30 * 1024 }), - (5849, EntryPoints::VectorPictureRead { length: 30 * 1024 }), - (35440, EntryPoints::SmartTablePicture { + }, + EntryPoints::InitializeVectorPicture { length: 40 }, + EntryPoints::VectorPicture { length: 40 }, + EntryPoints::VectorPictureRead { length: 40 }, + EntryPoints::InitializeVectorPicture { length: 30 * 1024 }, + EntryPoints::VectorPicture { length: 30 * 1024 }, + EntryPoints::VectorPictureRead { length: 30 * 1024 }, + EntryPoints::SmartTablePicture { length: 30 * 1024, num_points_per_txn: 200, - }), - (60464, EntryPoints::SmartTablePicture { + }, + EntryPoints::SmartTablePicture { length: 1024 * 1024, num_points_per_txn: 300, - }), - (13, EntryPoints::ResourceGroupsSenderWriteTag { + }, + EntryPoints::ResourceGroupsSenderWriteTag { string_length: 1024, - }), - (27, EntryPoints::ResourceGroupsSenderMultiChange { + }, + EntryPoints::ResourceGroupsSenderMultiChange { string_length: 1024, - }), - (291, EntryPoints::TokenV1MintAndTransferFT), - (468, EntryPoints::TokenV1MintAndTransferNFTSequential), - (386, EntryPoints::TokenV2AmbassadorMint { numbered: true }), - (467, EntryPoints::LiquidityPoolSwap { is_stable: true }), - (429, EntryPoints::LiquidityPoolSwap { is_stable: false }), + }, + EntryPoints::TokenV1MintAndTransferFT, + EntryPoints::TokenV1MintAndTransferNFTSequential, + EntryPoints::TokenV2AmbassadorMint { numbered: true }, + EntryPoints::LiquidityPoolSwap { is_stable: true }, + EntryPoints::LiquidityPoolSwap { is_stable: false }, ]; let mut failures = Vec::new(); @@ -154,7 +197,12 @@ fn main() { "wall time (us)", "expected (us)", "diff(- is impr)" ); - for (index, (expected_time, entry_point)) in entry_points.into_iter().enumerate() { + for (index, entry_point) in entry_points.into_iter().enumerate() { + let entry_point_name = format!("{:?}", entry_point); + let expected_time = calibration_values + .get(&entry_point_name) + .unwrap() + .expected_time; let publisher = executor.new_account_at(AccountAddress::random()); let mut package_handler = PackageHandler::new(entry_point.package_name()); @@ -189,23 +237,23 @@ fn main() { &package, publisher.address(), &mut executor, - if expected_time > 10000 { + if expected_time > 10000.0 { 6 - } else if expected_time > 1000 { + } else if expected_time > 1000.0 { 10 } else { 100 }, ); - let diff = (elapsed_micros as f32 - expected_time as f32) / (expected_time as f32) * 100.0; + let diff = (elapsed_micros as f32 - expected_time) / expected_time * 100.0; println!( - "{:15} {:15} {:14.1}% {:?}", + "{:15} {:15.1} {:14.1}% {:?}", elapsed_micros, expected_time, diff, entry_point ); json_lines.push(json!({ "grep": "grep_json_aptos_move_vm_perf", - "transaction_type": format!("{:?}", entry_point), + "transaction_type": entry_point_name, "wall_time_us": elapsed_micros, "expected_wall_time_us": expected_time, "test_index": index, diff --git a/execution/executor-benchmark/src/lib.rs b/execution/executor-benchmark/src/lib.rs index 428e404080df7f..80dd878e9682a0 100644 --- a/execution/executor-benchmark/src/lib.rs +++ b/execution/executor-benchmark/src/lib.rs @@ -770,7 +770,7 @@ fn log_total_supply(db_reader: &Arc) { mod tests { use crate::{ db_generator::bootstrap_with_genesis, init_db_and_executor, - native::native_config::NativeConfig, native_executor::NativeExecutor, + native::native_config::NativeConfig, pipeline::PipelineConfig, transaction_executor::BENCHMARKS_BLOCK_EXECUTOR_ONCHAIN_CONFIG, transaction_generator::TransactionGenerator, BenchmarkWorkload, }; @@ -833,14 +833,14 @@ mod tests { txn }; - let (_native_db, native_executor) = init_db_and_executor::(&config); - native_executor - .execute_and_state_checkpoint( - (HashValue::zero(), vec![txn]).into(), - native_executor.committed_block_id(), - BENCHMARKS_BLOCK_EXECUTOR_ONCHAIN_CONFIG, - ) - .unwrap(); + // let (_native_db, native_executor) = init_db_and_executor::(&config); + // native_executor + // .execute_and_state_checkpoint( + // (HashValue::zero(), vec![txn]).into(), + // native_executor.committed_block_id(), + // BENCHMARKS_BLOCK_EXECUTOR_ONCHAIN_CONFIG, + // ) + // .unwrap(); // let ( // vm_txns, @@ -955,9 +955,9 @@ mod tests { ); } - #[test] - fn test_native_benchmark() { - // correct execution not yet implemented, so cannot be checked for validity - test_generic_benchmark::(Some(TransactionTypeArg::AptFaTransfer), false); - } + // #[test] + // fn test_native_benchmark() { + // // correct execution not yet implemented, so cannot be checked for validity + // test_generic_benchmark::(None, false); + // } } diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 2bca6e61f10b33..afa4cb1ea076ef 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -50,7 +50,7 @@ class Flow(Flag): DEFAULT_NUM_INIT_ACCOUNTS = ( "100000000" if SELECTED_FLOW == Flow.MAINNET_LARGE_DB else "2000000" ) -DEFAULT_MAX_BLOCK_SIZE = "30000" +DEFAULT_MAX_BLOCK_SIZE = "10000" MAX_BLOCK_SIZE = int(os.environ.get("MAX_BLOCK_SIZE", default=DEFAULT_MAX_BLOCK_SIZE)) NUM_BLOCKS = int(os.environ.get("NUM_BLOCKS_PER_TEST", default=30)) @@ -73,9 +73,9 @@ class Flow(Flag): SKIP_WARNS = IS_MAINNET SKIP_PERF_IMPROVEMENT_NOTICE = IS_MAINNET -# bump after a perf improvement, so you can easily distinguish runs +# bump after a bigger test or perf change, so you can easily distinguish runs # that are on top of this commit -CODE_PERF_VERSION = "v6" +CODE_PERF_VERSION = "v7" # default to using production number of execution threads for assertions NUMBER_OF_EXECUTION_THREADS = int( @@ -161,47 +161,47 @@ class RunGroupConfig: # transaction_type module_working_set_size executor_type count min_ratio max_ratio median CALIBRATION = """ -no-op 1 VM 34 0.841 1.086 42046.2 -no-op 1000 VM 33 0.857 1.026 23125.1 -apt-fa-transfer 1 VM 34 0.843 1.057 29851.6 -apt-fa-transfer 1 NativeSpeculative 34 0.843 1.057 29851.6 -account-generation 1 VM 34 0.843 1.046 24134.9 -account-resource32-b 1 VM 34 0.803 1.089 37283.8 -modify-global-resource 1 VM 34 0.841 1.017 2854.7 -modify-global-resource 100 VM 34 0.844 1.035 36514.1 -publish-package 1 VM 34 0.915 1.049 143.4 -mix_publish_transfer 1 VM 34 0.912 1.131 2149.7 -batch100-transfer 1 VM 33 0.823 1.037 754.2 -vector-picture30k 1 VM 33 0.892 1.018 112.4 -vector-picture30k 100 VM 34 0.706 1.03 2050.1 -smart-table-picture30-k-with200-change 1 VM 34 0.959 1.057 21.5 -smart-table-picture30-k-with200-change 100 VM 34 0.9 1.021 412.2 -modify-global-resource-agg-v2 1 VM 34 0.729 1.076 39288.2 -modify-global-flag-agg-v2 1 VM 34 0.948 1.016 5598.2 -modify-global-bounded-agg-v2 1 VM 34 0.881 1.06 9968.4 -modify-global-milestone-agg-v2 1 VM 34 0.831 1.029 29575.5 -resource-groups-global-write-tag1-kb 1 VM 34 0.933 1.051 9285.8 -resource-groups-global-write-and-read-tag1-kb 1 VM 34 0.9 1.016 6353 -resource-groups-sender-write-tag1-kb 1 VM 34 0.845 1.163 20568.6 -resource-groups-sender-multi-change1-kb 1 VM 34 0.888 1.116 17029.7 -token-v1ft-mint-and-transfer 1 VM 34 0.853 1.029 1455.7 -token-v1ft-mint-and-transfer 100 VM 34 0.801 1.021 20418.7 -token-v1nft-mint-and-transfer-sequential 1 VM 34 0.881 1.023 884.4 -token-v1nft-mint-and-transfer-sequential 100 VM 34 0.85 1.021 14733.1 -coin-init-and-mint 1 VM 35 0.839 1.056 31116.2 -coin-init-and-mint 100 VM 35 0.788 1.04 25367 -fungible-asset-mint 1 VM 35 0.861 1.043 27493.2 -fungible-asset-mint 100 VM 35 0.865 1.033 22113.3 -no-op5-signers 1 VM 34 0.825 1.104 41817.6 -token-v2-ambassador-mint 1 VM 35 0.864 1.026 18187.6 -token-v2-ambassador-mint 100 VM 35 0.894 1.033 16597.8 -liquidity-pool-swap 1 VM 34 0.894 1.026 965 -liquidity-pool-swap 100 VM 35 0.893 1.026 11439.4 -liquidity-pool-swap-stable 1 VM 36 0.897 1.018 945.1 -liquidity-pool-swap-stable 100 VM 36 0.824 1.031 11196.4 -deserialize-u256 1 VM 36 0.881 1.06 41062.1 -no-op-fee-payer 1 VM 36 0.863 1.031 2141.3 -no-op-fee-payer 100 VM 36 0.898 1.02 28717.2 +no-op 1 VM 6 0.954 1.012 39580.1 +no-op 1000 VM 6 0.985 1.011 21446.3 +apt-fa-transfer 1 VM 6 0.966 1.031 26147.2 +account-generation 1 VM 6 0.974 1.039 21719.0 +account-resource32-b 1 VM 6 0.932 1.037 33261.7 +modify-global-resource 1 VM 6 0.991 1.017 2437.4 +modify-global-resource 100 VM 6 0.945 1.017 32670.2 +publish-package 1 VM 6 0.985 1.034 128.1 +mix_publish_transfer 1 VM 6 0.986 1.068 1724.5 +batch100-transfer 1 VM 6 0.920 1.044 726.5 +vector-picture30k 1 VM 6 0.980 1.018 98.2 +vector-picture30k 100 VM 6 0.859 1.050 1824.7 +smart-table-picture30-k-with200-change 1 VM 6 0.987 1.092 18.7 +smart-table-picture30-k-with200-change 100 VM 6 0.981 1.041 344.9 +modify-global-resource-agg-v2 1 VM 6 0.953 1.047 35548.1 +modify-global-flag-agg-v2 1 VM 6 0.983 1.027 4590.4 +modify-global-bounded-agg-v2 1 VM 6 0.934 1.021 8649.2 +modify-global-milestone-agg-v2 1 VM 6 0.925 1.049 24264.0 +resource-groups-global-write-tag1-kb 1 VM 6 0.911 1.066 9444.7 +resource-groups-global-write-and-read-tag1-kb 1 VM 6 0.857 1.033 5760.7 +resource-groups-sender-write-tag1-kb 1 VM 6 0.935 1.063 16564.9 +resource-groups-sender-multi-change1-kb 1 VM 6 0.970 1.031 15439.9 +token-v1ft-mint-and-transfer 1 VM 6 0.989 1.013 1264.5 +token-v1ft-mint-and-transfer 100 VM 6 0.913 1.034 17980.1 +token-v1nft-mint-and-transfer-sequential 1 VM 6 0.933 1.026 767.7 +token-v1nft-mint-and-transfer-sequential 100 VM 6 0.960 1.054 12415.7 +coin-init-and-mint 1 VM 6 0.921 1.030 27792.6 +coin-init-and-mint 100 VM 6 0.910 1.036 23457.3 +fungible-asset-mint 1 VM 6 0.926 1.016 23793.9 +fungible-asset-mint 100 VM 6 0.971 1.027 20158.2 +no-op5-signers 1 VM 6 0.960 1.025 39215.5 +token-v2-ambassador-mint 1 VM 6 0.962 1.006 16367.7 +token-v2-ambassador-mint 100 VM 6 0.969 1.019 15236.6 +liquidity-pool-swap 1 VM 6 0.989 1.026 859.2 +liquidity-pool-swap 100 VM 6 0.975 1.012 10378.5 +liquidity-pool-swap-stable 1 VM 6 0.981 1.024 820.0 +liquidity-pool-swap-stable 100 VM 6 0.980 1.025 10079.1 +deserialize-u256 1 VM 6 0.905 1.020 37213.4 +no-op-fee-payer 1 VM 6 0.976 1.011 1772.0 +no-op-fee-payer 100 VM 6 0.936 1.026 28981.5 +simple-script 1 VM 6 0.860 1.323 18361.5 """ # when adding a new test, add estimated expected_tps to it, as well as waived=True. @@ -286,6 +286,7 @@ class RunGroupConfig: # fee payer sequentializes transactions today. in these tests module publisher is the fee payer, so larger number of modules tests throughput with multiple fee payers RunGroupConfig(key=RunGroupKey("no-op-fee-payer"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(key=RunGroupKey("no-op-fee-payer", module_working_set_size=DEFAULT_MODULE_WORKING_SET_SIZE), included_in=Flow.CONTINUOUS), + RunGroupConfig(key=RunGroupKey("simple-script"), included_in=LAND_BLOCKING_AND_C, waived=True), RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True), RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True), @@ -298,7 +299,6 @@ class RunGroupConfig: # RunGroupConfig(expected_tps=17000 if NUM_ACCOUNTS < 5000000 else 28000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.MAINNET | Flow.MAINNET_LARGE_DB, waived=True), # RunGroupConfig(expected_tps=27000 if NUM_ACCOUNTS < 5000000 else 23000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.MAINNET | Flow.MAINNET_LARGE_DB, waived=True), - RunGroupConfig(expected_tps=1000, key=RunGroupKey("simple-script"), included_in=LAND_BLOCKING_AND_C, waived=True), ] # fmt: on @@ -688,7 +688,8 @@ def print_table( * pow(cur_calibration.max_ratio, 0.8), ) - cur_block_size = int(min([criteria.expected_tps, MAX_BLOCK_SIZE])) + # target 250ms blocks, a bit larger than prod + cur_block_size = max(4, int(min([criteria.expected_tps, MAX_BLOCK_SIZE]) / 4)) print(f"Testing {test.key}") if test.key_extra.transaction_type_override == "":