Skip to content

Commit

Permalink
update recalibration
Browse files Browse the repository at this point in the history
  • Loading branch information
@igor-aptos
igor-aptos committed Nov 7, 2024
1 parent 31336be commit cdf9d3a
Show file tree
Hide file tree
Showing 3 changed files with 149 additions and 100 deletions.
128 changes: 88 additions & 40 deletions aptos-move/e2e-benchmark/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use aptos_transaction_generator_lib::{
use aptos_types::{account_address::AccountAddress, transaction::TransactionPayload};
use rand::{rngs::StdRng, SeedableRng};
use serde_json::json;
use std::process::exit;
use std::{collections::HashMap, process::exit};

pub fn execute_txn(
executor: &mut FakeExecutor,
Expand Down Expand Up @@ -80,70 +80,113 @@ const ALLOWED_REGRESSION: f32 = 0.15;
const ALLOWED_IMPROVEMENT: f32 = 0.15;
const ABSOLUTE_BUFFER_US: f32 = 2.0;

const CALIBRATION_VALUES: &str = "
Loop { loop_count: Some(100000), loop_type: NoOp } 6 0.988 1.039 41212.4
Loop { loop_count: Some(10000), loop_type: Arithmetic } 6 0.977 1.038 25868.8
CreateObjects { num_objects: 10, object_payload_size: 0 } 6 0.940 1.026 152.1
CreateObjects { num_objects: 10, object_payload_size: 10240 } 6 0.934 1.051 9731.3
CreateObjects { num_objects: 100, object_payload_size: 0 } 6 0.966 1.051 1458.3
CreateObjects { num_objects: 100, object_payload_size: 10240 } 6 0.969 1.077 11196.4
InitializeVectorPicture { length: 40 } 6 0.973 1.066 75.0
VectorPicture { length: 40 } 6 0.955 1.092 22.0
VectorPictureRead { length: 40 } 6 0.952 1.047 21.0
InitializeVectorPicture { length: 30720 } 6 0.969 1.071 27295.8
VectorPicture { length: 30720 } 6 0.957 1.066 6560.2
VectorPictureRead { length: 30720 } 6 0.948 1.053 6642.8
SmartTablePicture { length: 30720, num_points_per_txn: 200 } 6 0.972 1.024 42660.4
SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 6 0.961 1.020 73725.5
ResourceGroupsSenderWriteTag { string_length: 1024 } 6 0.867 1.001 15.0
ResourceGroupsSenderMultiChange { string_length: 1024 } 6 0.966 1.069 29.0
TokenV1MintAndTransferFT 6 0.972 1.045 356.8
TokenV1MintAndTransferNFTSequential 6 0.991 1.067 543.7
TokenV2AmbassadorMint { numbered: true } 6 0.987 1.052 474.4
LiquidityPoolSwap { is_stable: true } 6 0.970 1.042 555.4
LiquidityPoolSwap { is_stable: false } 6 0.925 1.001 535.3
";

struct CalibrationInfo {
// count: usize,
expected_time: f32,
}

fn get_parsed_calibration_values() -> HashMap<String, CalibrationInfo> {
CALIBRATION_VALUES
.trim()
.split('\n')
.map(|line| {
let parts = line.split('\t').collect::<Vec<_>>();
(parts[0].to_string(), CalibrationInfo {
// count: parts[1].parse().unwrap(),
expected_time: parts[parts.len() - 1].parse().unwrap(),
})
})
.collect()
}

fn main() {
let executor = FakeExecutor::from_head_genesis();
let mut executor = executor.set_not_parallel();

let calibration_values = get_parsed_calibration_values();

let entry_points = vec![
// too fast for the timer
// (, EntryPoints::Nop),
// (, EntryPoints::BytesMakeOrChange {
// data_length: Some(32),
// }),
// (, EntryPoints::IncGlobal),
(34651, EntryPoints::Loop {
EntryPoints::Loop {
loop_count: Some(100000),
loop_type: LoopType::NoOp,
}),
(21145, EntryPoints::Loop {
},
EntryPoints::Loop {
loop_count: Some(10000),
loop_type: LoopType::Arithmetic,
}),
},
// This is a cheap bcs (serializing vec<u8>), so not representative of what BCS native call should cost.
// (, EntryPoints::Loop { loop_count: Some(1000), loop_type: LoopType::BCS { len: 1024 }}),
(124, EntryPoints::CreateObjects {
EntryPoints::CreateObjects {
num_objects: 10,
object_payload_size: 0,
}),
(8090, EntryPoints::CreateObjects {
},
EntryPoints::CreateObjects {
num_objects: 10,
object_payload_size: 10 * 1024,
}),
(1246, EntryPoints::CreateObjects {
},
EntryPoints::CreateObjects {
num_objects: 100,
object_payload_size: 0,
}),
(9556, EntryPoints::CreateObjects {
},
EntryPoints::CreateObjects {
num_objects: 100,
object_payload_size: 10 * 1024,
}),
(61, EntryPoints::InitializeVectorPicture { length: 40 }),
(16, EntryPoints::VectorPicture { length: 40 }),
(16, EntryPoints::VectorPictureRead { length: 40 }),
(23256, EntryPoints::InitializeVectorPicture {
length: 30 * 1024,
}),
(5860, EntryPoints::VectorPicture { length: 30 * 1024 }),
(5849, EntryPoints::VectorPictureRead { length: 30 * 1024 }),
(35440, EntryPoints::SmartTablePicture {
},
EntryPoints::InitializeVectorPicture { length: 40 },
EntryPoints::VectorPicture { length: 40 },
EntryPoints::VectorPictureRead { length: 40 },
EntryPoints::InitializeVectorPicture { length: 30 * 1024 },
EntryPoints::VectorPicture { length: 30 * 1024 },
EntryPoints::VectorPictureRead { length: 30 * 1024 },
EntryPoints::SmartTablePicture {
length: 30 * 1024,
num_points_per_txn: 200,
}),
(60464, EntryPoints::SmartTablePicture {
},
EntryPoints::SmartTablePicture {
length: 1024 * 1024,
num_points_per_txn: 300,
}),
(13, EntryPoints::ResourceGroupsSenderWriteTag {
},
EntryPoints::ResourceGroupsSenderWriteTag {
string_length: 1024,
}),
(27, EntryPoints::ResourceGroupsSenderMultiChange {
},
EntryPoints::ResourceGroupsSenderMultiChange {
string_length: 1024,
}),
(291, EntryPoints::TokenV1MintAndTransferFT),
(468, EntryPoints::TokenV1MintAndTransferNFTSequential),
(386, EntryPoints::TokenV2AmbassadorMint { numbered: true }),
(467, EntryPoints::LiquidityPoolSwap { is_stable: true }),
(429, EntryPoints::LiquidityPoolSwap { is_stable: false }),
},
EntryPoints::TokenV1MintAndTransferFT,
EntryPoints::TokenV1MintAndTransferNFTSequential,
EntryPoints::TokenV2AmbassadorMint { numbered: true },
EntryPoints::LiquidityPoolSwap { is_stable: true },
EntryPoints::LiquidityPoolSwap { is_stable: false },
];

let mut failures = Vec::new();
Expand All @@ -154,7 +197,12 @@ fn main() {
"wall time (us)", "expected (us)", "diff(- is impr)"
);

for (index, (expected_time, entry_point)) in entry_points.into_iter().enumerate() {
for (index, entry_point) in entry_points.into_iter().enumerate() {
let entry_point_name = format!("{:?}", entry_point);
let expected_time = calibration_values
.get(&entry_point_name)
.unwrap()
.expected_time;
let publisher = executor.new_account_at(AccountAddress::random());

let mut package_handler = PackageHandler::new(entry_point.package_name());
Expand Down Expand Up @@ -189,23 +237,23 @@ fn main() {
&package,
publisher.address(),
&mut executor,
if expected_time > 10000 {
if expected_time > 10000.0 {
6
} else if expected_time > 1000 {
} else if expected_time > 1000.0 {
10
} else {
100
},
);
let diff = (elapsed_micros as f32 - expected_time as f32) / (expected_time as f32) * 100.0;
let diff = (elapsed_micros as f32 - expected_time) / expected_time * 100.0;
println!(
"{:15} {:15} {:14.1}% {:?}",
"{:15} {:15.1} {:14.1}% {:?}",
elapsed_micros, expected_time, diff, entry_point
);

json_lines.push(json!({
"grep": "grep_json_aptos_move_vm_perf",
"transaction_type": format!("{:?}", entry_point),
"transaction_type": entry_point_name,
"wall_time_us": elapsed_micros,
"expected_wall_time_us": expected_time,
"test_index": index,
Expand Down
28 changes: 14 additions & 14 deletions execution/executor-benchmark/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -770,7 +770,7 @@ fn log_total_supply(db_reader: &Arc<dyn DbReader>) {
mod tests {
use crate::{
db_generator::bootstrap_with_genesis, init_db_and_executor,
native::native_config::NativeConfig, native_executor::NativeExecutor,
native::native_config::NativeConfig,
pipeline::PipelineConfig, transaction_executor::BENCHMARKS_BLOCK_EXECUTOR_ONCHAIN_CONFIG,
transaction_generator::TransactionGenerator, BenchmarkWorkload,
};
Expand Down Expand Up @@ -833,14 +833,14 @@ mod tests {
txn
};

let (_native_db, native_executor) = init_db_and_executor::<NativeExecutor>(&config);
native_executor
.execute_and_state_checkpoint(
(HashValue::zero(), vec![txn]).into(),
native_executor.committed_block_id(),
BENCHMARKS_BLOCK_EXECUTOR_ONCHAIN_CONFIG,
)
.unwrap();
// let (_native_db, native_executor) = init_db_and_executor::<NativeExecutor>(&config);
// native_executor
// .execute_and_state_checkpoint(
// (HashValue::zero(), vec![txn]).into(),
// native_executor.committed_block_id(),
// BENCHMARKS_BLOCK_EXECUTOR_ONCHAIN_CONFIG,
// )
// .unwrap();

// let (
// vm_txns,
Expand Down Expand Up @@ -955,9 +955,9 @@ mod tests {
);
}

#[test]
fn test_native_benchmark() {
// correct execution not yet implemented, so cannot be checked for validity
test_generic_benchmark::<NativeExecutor>(Some(TransactionTypeArg::AptFaTransfer), false);
}
// #[test]
// fn test_native_benchmark() {
// // correct execution not yet implemented, so cannot be checked for validity
// test_generic_benchmark::<NativeExecutor>(None, false);
// }
}
93 changes: 47 additions & 46 deletions testsuite/single_node_performance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ class Flow(Flag):
DEFAULT_NUM_INIT_ACCOUNTS = (
"100000000" if SELECTED_FLOW == Flow.MAINNET_LARGE_DB else "2000000"
)
DEFAULT_MAX_BLOCK_SIZE = "30000"
DEFAULT_MAX_BLOCK_SIZE = "10000"

MAX_BLOCK_SIZE = int(os.environ.get("MAX_BLOCK_SIZE", default=DEFAULT_MAX_BLOCK_SIZE))
NUM_BLOCKS = int(os.environ.get("NUM_BLOCKS_PER_TEST", default=30))
Expand All @@ -73,9 +73,9 @@ class Flow(Flag):
SKIP_WARNS = IS_MAINNET
SKIP_PERF_IMPROVEMENT_NOTICE = IS_MAINNET

# bump after a perf improvement, so you can easily distinguish runs
# bump after a bigger test or perf change, so you can easily distinguish runs
# that are on top of this commit
CODE_PERF_VERSION = "v6"
CODE_PERF_VERSION = "v7"

# default to using production number of execution threads for assertions
NUMBER_OF_EXECUTION_THREADS = int(
Expand Down Expand Up @@ -161,47 +161,47 @@ class RunGroupConfig:

# transaction_type module_working_set_size executor_type count min_ratio max_ratio median
CALIBRATION = """
no-op 1 VM 34 0.841 1.086 42046.2
no-op 1000 VM 33 0.857 1.026 23125.1
apt-fa-transfer 1 VM 34 0.843 1.057 29851.6
apt-fa-transfer 1 NativeSpeculative 34 0.843 1.057 29851.6
account-generation 1 VM 34 0.843 1.046 24134.9
account-resource32-b 1 VM 34 0.803 1.089 37283.8
modify-global-resource 1 VM 34 0.841 1.017 2854.7
modify-global-resource 100 VM 34 0.844 1.035 36514.1
publish-package 1 VM 34 0.915 1.049 143.4
mix_publish_transfer 1 VM 34 0.912 1.131 2149.7
batch100-transfer 1 VM 33 0.823 1.037 754.2
vector-picture30k 1 VM 33 0.892 1.018 112.4
vector-picture30k 100 VM 34 0.706 1.03 2050.1
smart-table-picture30-k-with200-change 1 VM 34 0.959 1.057 21.5
smart-table-picture30-k-with200-change 100 VM 34 0.9 1.021 412.2
modify-global-resource-agg-v2 1 VM 34 0.729 1.076 39288.2
modify-global-flag-agg-v2 1 VM 34 0.948 1.016 5598.2
modify-global-bounded-agg-v2 1 VM 34 0.881 1.06 9968.4
modify-global-milestone-agg-v2 1 VM 34 0.831 1.029 29575.5
resource-groups-global-write-tag1-kb 1 VM 34 0.933 1.051 9285.8
resource-groups-global-write-and-read-tag1-kb 1 VM 34 0.9 1.016 6353
resource-groups-sender-write-tag1-kb 1 VM 34 0.845 1.163 20568.6
resource-groups-sender-multi-change1-kb 1 VM 34 0.888 1.116 17029.7
token-v1ft-mint-and-transfer 1 VM 34 0.853 1.029 1455.7
token-v1ft-mint-and-transfer 100 VM 34 0.801 1.021 20418.7
token-v1nft-mint-and-transfer-sequential 1 VM 34 0.881 1.023 884.4
token-v1nft-mint-and-transfer-sequential 100 VM 34 0.85 1.021 14733.1
coin-init-and-mint 1 VM 35 0.839 1.056 31116.2
coin-init-and-mint 100 VM 35 0.788 1.04 25367
fungible-asset-mint 1 VM 35 0.861 1.043 27493.2
fungible-asset-mint 100 VM 35 0.865 1.033 22113.3
no-op5-signers 1 VM 34 0.825 1.104 41817.6
token-v2-ambassador-mint 1 VM 35 0.864 1.026 18187.6
token-v2-ambassador-mint 100 VM 35 0.894 1.033 16597.8
liquidity-pool-swap 1 VM 34 0.894 1.026 965
liquidity-pool-swap 100 VM 35 0.893 1.026 11439.4
liquidity-pool-swap-stable 1 VM 36 0.897 1.018 945.1
liquidity-pool-swap-stable 100 VM 36 0.824 1.031 11196.4
deserialize-u256 1 VM 36 0.881 1.06 41062.1
no-op-fee-payer 1 VM 36 0.863 1.031 2141.3
no-op-fee-payer 100 VM 36 0.898 1.02 28717.2
no-op 1 VM 6 0.954 1.012 39580.1
no-op 1000 VM 6 0.985 1.011 21446.3
apt-fa-transfer 1 VM 6 0.966 1.031 26147.2
account-generation 1 VM 6 0.974 1.039 21719.0
account-resource32-b 1 VM 6 0.932 1.037 33261.7
modify-global-resource 1 VM 6 0.991 1.017 2437.4
modify-global-resource 100 VM 6 0.945 1.017 32670.2
publish-package 1 VM 6 0.985 1.034 128.1
mix_publish_transfer 1 VM 6 0.986 1.068 1724.5
batch100-transfer 1 VM 6 0.920 1.044 726.5
vector-picture30k 1 VM 6 0.980 1.018 98.2
vector-picture30k 100 VM 6 0.859 1.050 1824.7
smart-table-picture30-k-with200-change 1 VM 6 0.987 1.092 18.7
smart-table-picture30-k-with200-change 100 VM 6 0.981 1.041 344.9
modify-global-resource-agg-v2 1 VM 6 0.953 1.047 35548.1
modify-global-flag-agg-v2 1 VM 6 0.983 1.027 4590.4
modify-global-bounded-agg-v2 1 VM 6 0.934 1.021 8649.2
modify-global-milestone-agg-v2 1 VM 6 0.925 1.049 24264.0
resource-groups-global-write-tag1-kb 1 VM 6 0.911 1.066 9444.7
resource-groups-global-write-and-read-tag1-kb 1 VM 6 0.857 1.033 5760.7
resource-groups-sender-write-tag1-kb 1 VM 6 0.935 1.063 16564.9
resource-groups-sender-multi-change1-kb 1 VM 6 0.970 1.031 15439.9
token-v1ft-mint-and-transfer 1 VM 6 0.989 1.013 1264.5
token-v1ft-mint-and-transfer 100 VM 6 0.913 1.034 17980.1
token-v1nft-mint-and-transfer-sequential 1 VM 6 0.933 1.026 767.7
token-v1nft-mint-and-transfer-sequential 100 VM 6 0.960 1.054 12415.7
coin-init-and-mint 1 VM 6 0.921 1.030 27792.6
coin-init-and-mint 100 VM 6 0.910 1.036 23457.3
fungible-asset-mint 1 VM 6 0.926 1.016 23793.9
fungible-asset-mint 100 VM 6 0.971 1.027 20158.2
no-op5-signers 1 VM 6 0.960 1.025 39215.5
token-v2-ambassador-mint 1 VM 6 0.962 1.006 16367.7
token-v2-ambassador-mint 100 VM 6 0.969 1.019 15236.6
liquidity-pool-swap 1 VM 6 0.989 1.026 859.2
liquidity-pool-swap 100 VM 6 0.975 1.012 10378.5
liquidity-pool-swap-stable 1 VM 6 0.981 1.024 820.0
liquidity-pool-swap-stable 100 VM 6 0.980 1.025 10079.1
deserialize-u256 1 VM 6 0.905 1.020 37213.4
no-op-fee-payer 1 VM 6 0.976 1.011 1772.0
no-op-fee-payer 100 VM 6 0.936 1.026 28981.5
simple-script 1 VM 6 0.860 1.323 18361.5
"""

# when adding a new test, add estimated expected_tps to it, as well as waived=True.
Expand Down Expand Up @@ -286,6 +286,7 @@ class RunGroupConfig:
# fee payer sequentializes transactions today. in these tests module publisher is the fee payer, so larger number of modules tests throughput with multiple fee payers
RunGroupConfig(key=RunGroupKey("no-op-fee-payer"), included_in=LAND_BLOCKING_AND_C),
RunGroupConfig(key=RunGroupKey("no-op-fee-payer", module_working_set_size=DEFAULT_MODULE_WORKING_SET_SIZE), included_in=Flow.CONTINUOUS),
RunGroupConfig(key=RunGroupKey("simple-script"), included_in=LAND_BLOCKING_AND_C, waived=True),

RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True),
RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True),
Expand All @@ -298,7 +299,6 @@ class RunGroupConfig:
# RunGroupConfig(expected_tps=17000 if NUM_ACCOUNTS < 5000000 else 28000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.MAINNET | Flow.MAINNET_LARGE_DB, waived=True),
# RunGroupConfig(expected_tps=27000 if NUM_ACCOUNTS < 5000000 else 23000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.MAINNET | Flow.MAINNET_LARGE_DB, waived=True),

RunGroupConfig(expected_tps=1000, key=RunGroupKey("simple-script"), included_in=LAND_BLOCKING_AND_C, waived=True),
]
# fmt: on

Expand Down Expand Up @@ -688,7 +688,8 @@ def print_table(
* pow(cur_calibration.max_ratio, 0.8),
)

cur_block_size = int(min([criteria.expected_tps, MAX_BLOCK_SIZE]))
# target 250ms blocks, a bit larger than prod
cur_block_size = max(4, int(min([criteria.expected_tps, MAX_BLOCK_SIZE]) / 4))

print(f"Testing {test.key}")
if test.key_extra.transaction_type_override == "":
Expand Down

0 comments on commit cdf9d3a

Please sign in to comment.