From de855b72d4e87fbf78bf68f8bd30b07be271fb8b Mon Sep 17 00:00:00 2001 From: quellen-sol Date: Thu, 11 Jan 2024 12:03:49 -0800 Subject: [PATCH] Step Finance patch Co-authored-by: Dana bump docker build version Add account owners array to messages Cleaner diff rename to post_owners --- .circleci/config.yml | 25 ++++++ _step-finance/Dockerfile | 14 ++++ _step-finance/Dockerfile.dockerignore | 2 + core/src/banking_stage/committer.rs | 11 ++- core/src/banking_stage/consumer.rs | 4 +- ledger/src/blockstore_processor.rs | 14 +++- rpc-client/src/mock_sender.rs | 3 + rpc/src/transaction_status_service.rs | 21 ++++- runtime/src/bank.rs | 108 ++++++++++++++++++++++++-- runtime/src/bank/tests.rs | 2 +- runtime/src/lib.rs | 1 + runtime/src/program_inclusions.rs | 73 +++++++++++++++++ runtime/src/runtime_config.rs | 5 ++ storage-bigtable/src/lib.rs | 3 + storage-proto/src/convert.rs | 4 + storage-proto/src/lib.rs | 4 + test-validator/src/lib.rs | 8 +- transaction-status/src/lib.rs | 71 ++++++++++++++++- validator/src/main.rs | 6 ++ 19 files changed, 363 insertions(+), 16 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 _step-finance/Dockerfile create mode 100644 _step-finance/Dockerfile.dockerignore create mode 100644 runtime/src/program_inclusions.rs diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000000..f4bc0528e11b2d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,25 @@ +version: 2.1 + +orbs: + aws-ecr: circleci/aws-ecr@8.1.3 + +executors: + custom: + machine: + image: ubuntu-2204:2022.04.2 + resource_class: xlarge + +workflows: + run-pipeline: + jobs: + - aws-ecr/build-and-push-image: + executor: custom + dockerfile: _step-finance/Dockerfile + repo: step-solana + region: eu-north-1 + tag: ${CIRCLE_BRANCH},${CIRCLE_BRANCH}-<< pipeline.number >>,${CIRCLE_BRANCH}-<< pipeline.number >>-${CIRCLE_SHA1} + context: aws-account + filters: + branches: + only: + - step-release diff --git a/_step-finance/Dockerfile b/_step-finance/Dockerfile new file mode 100644 index 00000000000000..2edf8d4eb77ac3 --- /dev/null +++ b/_step-finance/Dockerfile @@ -0,0 +1,14 @@ +FROM rust:1.73-bullseye as builder +RUN apt-get update +RUN apt-get install -y libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang cmake make libprotobuf-dev protobuf-compiler +WORKDIR /solana-build +COPY . . +RUN cargo build --release + +FROM debian:bullseye-20230522 as runtime +RUN apt-get update +RUN apt-get install -y bzip2 +COPY --from=builder --chmod=700 /solana-build/target/release /usr/local/bin + +WORKDIR /solana +ENTRYPOINT ["solana-test-validator"] diff --git a/_step-finance/Dockerfile.dockerignore b/_step-finance/Dockerfile.dockerignore new file mode 100644 index 00000000000000..fdab7b85601aec --- /dev/null +++ b/_step-finance/Dockerfile.dockerignore @@ -0,0 +1,2 @@ +target/ +_step-finance/ diff --git a/core/src/banking_stage/committer.rs b/core/src/banking_stage/committer.rs index ab8f3a9ed57b5b..86cd5825cbda55 100644 --- a/core/src/banking_stage/committer.rs +++ b/core/src/banking_stage/committer.rs @@ -10,9 +10,10 @@ use { }, solana_measure::measure_us, solana_runtime::{ - bank::{Bank, CommitTransactionCounts, TransactionBalancesSet}, + bank::{Bank, CommitTransactionCounts, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet}, bank_utils, prioritization_fee_cache::PrioritizationFeeCache, + program_inclusions::PreOrPostDatum, transaction_batch::TransactionBatch, }, solana_sdk::{hash::Hash, pubkey::Pubkey, saturating_add_assign}, @@ -32,6 +33,7 @@ pub enum CommitTransactionDetails { #[derive(Default)] pub(super) struct PreBalanceInfo { pub native: Vec>, + pub datum: Vec>>>, pub token: Vec>, pub mint_decimals: HashMap, } @@ -142,7 +144,8 @@ impl Committer { ) { if let Some(transaction_status_sender) = &self.transaction_status_sender { let txs = batch.sanitized_transactions().to_vec(); - let post_balances = bank.collect_balances(batch); + let (post_balances, post_datum, owners) = + bank.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum); let post_token_balances = collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals); let mut transaction_index = starting_transaction_index.unwrap_or_default(); @@ -167,6 +170,10 @@ impl Committer { std::mem::take(&mut pre_balance_info.native), post_balances, ), + TransactionOwnersSet { + owners, + }, + TransactionDatumSet::new(std::mem::take(&mut pre_balance_info.datum), post_datum), TransactionTokenBalancesSet::new( std::mem::take(&mut pre_balance_info.token), post_token_balances, diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 1b8487a923b972..c716b140789e2d 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -25,6 +25,7 @@ use { solana_runtime::{ accounts::validate_fee_payer, bank::{Bank, LoadAndExecuteTransactionsOutput}, + program_inclusions::PreOrPostDatum, transaction_batch::TransactionBatch, }, solana_sdk::{ @@ -570,7 +571,8 @@ impl Consumer { // If the extra meta-data services are enabled for RPC, collect the // pre-balances for native and token programs. if transaction_status_sender_enabled { - pre_balance_info.native = bank.collect_balances(batch); + (pre_balance_info.native, pre_balance_info.datum, ..) = + bank.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum); pre_balance_info.token = collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals) } diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 4e093814b9bb3a..2749d5ad66d4c2 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -35,7 +35,7 @@ use { solana_rayon_threadlimit::{get_max_thread_count, get_thread_count}, solana_runtime::{ accounts_background_service::{AbsRequestSender, SnapshotRequestKind}, - bank::{Bank, TransactionBalancesSet}, + bank::{Bank, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet}, bank_forks::BankForks, bank_utils, commitment::VOTE_THRESHOLD_SIZE, @@ -157,7 +157,7 @@ pub fn execute_batch( vec![] }; - let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions( + let (tx_results, balances, datum, owners) = batch.bank().load_execute_and_commit_transactions( batch, MAX_PROCESSING_AGE, transaction_status_sender.is_some(), @@ -203,6 +203,8 @@ pub fn execute_batch( transactions, execution_results, balances, + owners, + datum, token_balances, rent_debits, transaction_indexes.to_vec(), @@ -1828,6 +1830,8 @@ pub struct TransactionStatusBatch { pub transactions: Vec, pub execution_results: Vec>, pub balances: TransactionBalancesSet, + pub owners: TransactionOwnersSet, + pub datum: TransactionDatumSet, pub token_balances: TransactionTokenBalancesSet, pub rent_debits: Vec, pub transaction_indexes: Vec, @@ -1845,6 +1849,8 @@ impl TransactionStatusSender { transactions: Vec, execution_results: Vec, balances: TransactionBalancesSet, + owners: TransactionOwnersSet, + datum: TransactionDatumSet, token_balances: TransactionTokenBalancesSet, rent_debits: Vec, transaction_indexes: Vec, @@ -1864,6 +1870,8 @@ impl TransactionStatusSender { }) .collect(), balances, + owners, + datum, token_balances, rent_debits, transaction_indexes, @@ -3951,6 +3959,8 @@ pub mod tests { .. }, _balances, + _datums, + _owners, ) = batch.bank().load_execute_and_commit_transactions( &batch, MAX_PROCESSING_AGE, diff --git a/rpc-client/src/mock_sender.rs b/rpc-client/src/mock_sender.rs index 44ab26359c3f32..cf86f318caea8d 100644 --- a/rpc-client/src/mock_sender.rs +++ b/rpc-client/src/mock_sender.rs @@ -226,6 +226,9 @@ impl RpcSender for MockSender { fee: 0, pre_balances: vec![499999999999999950, 50, 1], post_balances: vec![499999999999999950, 50, 1], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: OptionSerializer::None, log_messages: OptionSerializer::None, pre_token_balances: OptionSerializer::None, diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index eca53c66658766..28b5462ebd103d 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -69,6 +69,8 @@ impl TransactionStatusService { bank, transactions, execution_results, + owners, + datum, balances, token_balances, rent_debits, @@ -80,6 +82,9 @@ impl TransactionStatusService { execution_result, pre_balances, post_balances, + owners, + pre_datum, + post_datum, pre_token_balances, post_token_balances, rent_debits, @@ -89,6 +94,9 @@ impl TransactionStatusService { execution_results, balances.pre_balances, balances.post_balances, + owners.owners, + datum.pre_datum, + datum.post_datum, token_balances.pre_token_balances, token_balances.post_token_balances, rent_debits, @@ -144,6 +152,9 @@ impl TransactionStatusService { fee, pre_balances, post_balances, + post_owners: Some(owners), + pre_datum: Some(pre_datum), + post_datum: Some(post_datum), inner_instructions, log_messages, pre_token_balances, @@ -206,6 +217,7 @@ impl TransactionStatusService { #[cfg(test)] pub(crate) mod tests { + use { super::*, crate::transaction_notifier_interface::TransactionNotifier, @@ -217,7 +229,9 @@ pub(crate) mod tests { rent_debits::RentDebits, }, solana_ledger::{genesis_utils::create_genesis_config, get_tmp_ledger_path_auto_delete}, - solana_runtime::bank::{Bank, TransactionBalancesSet}, + solana_runtime::bank::{ + Bank, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet, + }, solana_sdk::{ account_utils::StateMut, clock::Slot, @@ -410,6 +424,11 @@ pub(crate) mod tests { let transaction_index: usize = bank.transaction_count().try_into().unwrap(); let transaction_status_batch = TransactionStatusBatch { bank, + owners: TransactionOwnersSet { owners: vec![] }, + datum: TransactionDatumSet { + post_datum: vec![vec![Some(vec![0x69])]], + pre_datum: vec![vec![Some(vec![0x04, 0x20])]], + }, transactions: vec![transaction], execution_results: vec![transaction_result], balances, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 8b217d3b8cf679..ba55eb256891d9 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -48,6 +48,7 @@ use { epoch_rewards_hasher::hash_rewards_into_partitions, epoch_stakes::{EpochStakes, NodeVoteAccounts}, installed_scheduler_pool::{BankWithScheduler, InstalledSchedulerRwLock}, + program_inclusions::{PreOrPostDatum, ProgramDatumInclusions}, runtime_config::RuntimeConfig, serde_snapshot::BankIncrementalSnapshotPersistence, snapshot_hash::SnapshotHash, @@ -355,6 +356,27 @@ impl TransactionBalancesSet { } pub type TransactionBalances = Vec>; +pub struct TransactionDatumSet { + pub pre_datum: TransactionDatum, + pub post_datum: TransactionDatum, +} + +impl TransactionDatumSet { + pub fn new(pre_datum: TransactionDatum, post_datum: TransactionDatum) -> Self { + Self { + pre_datum, + post_datum, + } + } +} +pub type TransactionDatum = Vec>>>; + +pub struct TransactionOwnersSet { + pub owners: TransactionOwners, +} + +pub type TransactionOwners = Vec>>; + /// A list of log messages emitted during a transaction pub type TransactionLogMessages = Vec; @@ -570,6 +592,7 @@ impl PartialEq for Bank { loaded_programs_cache: _, check_program_modification_slot: _, epoch_reward_status: _, + program_datum_inclusions: _, // Ignore new fields explicitly if they do not impact PartialEq. // Adding ".." will remove compile-time checks that if a new field // is added to the struct, this PartialEq is accordingly updated. @@ -833,6 +856,8 @@ pub struct Bank { pub check_program_modification_slot: bool, epoch_reward_status: EpochRewardStatus, + /// programs that will have their account data sent to geyser + pub program_datum_inclusions: Arc, } struct VoteWithStakeDelegations { @@ -1020,6 +1045,7 @@ impl Bank { ))), check_program_modification_slot: false, epoch_reward_status: EpochRewardStatus::default(), + program_datum_inclusions: Arc::::default(), }; let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64; @@ -1054,6 +1080,7 @@ impl Bank { ); let accounts = Accounts::new(Arc::new(accounts_db)); let mut bank = Self::default_with_accounts(accounts); + bank.program_datum_inclusions = runtime_config.program_datum_inclusions.clone(); bank.ancestors = Ancestors::from(vec![bank.slot()]); bank.transaction_debug_keys = debug_keys; bank.runtime_config = runtime_config; @@ -1331,6 +1358,7 @@ impl Bank { loaded_programs_cache: parent.loaded_programs_cache.clone(), check_program_modification_slot: false, epoch_reward_status: parent.epoch_reward_status.clone(), + program_datum_inclusions: parent.program_datum_inclusions.clone(), }; let (_, ancestors_time_us) = measure_us!({ @@ -1767,6 +1795,7 @@ impl Bank { a corrupted snapshot or bugs in cached accounts or accounts-db.", ); let stakes_accounts_load_duration = now.elapsed(); + let program_datum_inclusions = runtime_config.program_datum_inclusions.clone(); let mut bank = Self { skipped_rewrites: Mutex::default(), incremental_snapshot_persistence: fields.incremental_snapshot_persistence, @@ -1833,6 +1862,7 @@ impl Bank { ))), check_program_modification_slot: false, epoch_reward_status: fields.epoch_reward_status, + program_datum_inclusions, }; bank.finish_init( genesis_config, @@ -4532,6 +4562,31 @@ impl Bank { balances } + pub fn collect_balances_and_datum( + &self, + batch: &TransactionBatch, + pre_or_post: PreOrPostDatum, + ) -> (TransactionBalances, TransactionDatum, TransactionOwners) { + let mut balances: TransactionBalances = vec![]; + let mut datum: TransactionDatum = vec![]; + let mut owners: TransactionOwners = vec![]; + for transaction in batch.sanitized_transactions() { + let mut transaction_balances: Vec = vec![]; + let mut transaction_datum: Vec>> = vec![]; + let mut transaction_owners: Vec> = vec![]; + for account_key in transaction.message().account_keys().iter() { + let (balance, data, owner) = self.get_balance_and_data(account_key, &pre_or_post); + transaction_balances.push(balance); + transaction_datum.push(data); + transaction_owners.push(owner); + } + balances.push(transaction_balances); + datum.push(transaction_datum); + owners.push(transaction_owners); + } + (balances, datum, owners) + } + fn program_modification_slot(&self, pubkey: &Pubkey) -> Result { let program = self .get_account_with_fixed_root(pubkey) @@ -6293,11 +6348,16 @@ impl Bank { enable_return_data_recording: bool, timings: &mut ExecuteTimings, log_messages_bytes_limit: Option, - ) -> (TransactionResults, TransactionBalancesSet) { - let pre_balances = if collect_balances { - self.collect_balances(batch) + ) -> ( + TransactionResults, + TransactionBalancesSet, + TransactionDatumSet, + TransactionOwnersSet, + ) { + let (pre_balances, pre_datum, ..) = if collect_balances { + self.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum) } else { - vec![] + (vec![], vec![], vec![]) }; let LoadAndExecuteTransactionsOutput { @@ -6338,14 +6398,16 @@ impl Bank { }, timings, ); - let post_balances = if collect_balances { - self.collect_balances(batch) + let (post_balances, post_datum, owners) = if collect_balances { + self.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum) } else { - vec![] + (vec![], vec![], vec![]) }; ( results, TransactionBalancesSet::new(pre_balances, post_balances), + TransactionDatumSet::new(pre_datum, post_datum), + TransactionOwnersSet { owners }, ) } @@ -6441,6 +6503,23 @@ impl Bank { pub fn read_balance(account: &AccountSharedData) -> u64 { account.lamports() } + + pub fn read_data( + &self, + account: &AccountSharedData, + pre_or_post: &PreOrPostDatum, + ) -> Option> { + let data = account.data(); + let owner = account.owner(); + let inclusion = self.program_datum_inclusions.get(owner)?; + let include_data = inclusion.can_include_datum(pre_or_post, &data); + + if !include_data { + None + } else { + Some(data.to_vec()) + } + } /// Each program would need to be able to introspect its own state /// this is hard-coded to the Budget language pub fn get_balance(&self, pubkey: &Pubkey) -> u64 { @@ -6448,6 +6527,21 @@ impl Bank { .map(|x| Self::read_balance(&x)) .unwrap_or(0) } + pub fn get_balance_and_data( + &self, + pubkey: &Pubkey, + pre_or_post: &PreOrPostDatum, + ) -> (u64, Option>, Option) { + self.get_account(pubkey) + .map(|x| { + ( + Self::read_balance(&x), + Self::read_data(self, &x, pre_or_post), + Some(*x.owner()), + ) + }) + .unwrap_or((0, None, None)) + } /// Compute all the parents of the bank in order pub fn parents(&self) -> Vec> { diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 18c9727a991d73..ae5ca7f8bf6456 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -5941,7 +5941,7 @@ fn test_pre_post_transaction_balances() { let txs = vec![tx0, tx1, tx2]; let lock_result = bank0.prepare_batch_for_tests(txs); - let (transaction_results, transaction_balances_set) = bank0 + let (transaction_results, transaction_balances_set, _) = bank0 .load_execute_and_commit_transactions( &lock_result, MAX_PROCESSING_AGE, diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index b0884a6f185c20..93b5b155d5b6e9 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -21,6 +21,7 @@ pub mod loader_utils; pub mod non_circulating_supply; pub mod prioritization_fee; pub mod prioritization_fee_cache; +pub mod program_inclusions; pub mod root_bank_cache; pub mod runtime_config; pub mod serde_snapshot; diff --git a/runtime/src/program_inclusions.rs b/runtime/src/program_inclusions.rs new file mode 100644 index 00000000000000..e4c6c21bdee66d --- /dev/null +++ b/runtime/src/program_inclusions.rs @@ -0,0 +1,73 @@ +use std::{collections::HashMap, fs, path::PathBuf}; + +use solana_sdk::pubkey::Pubkey; + +pub enum PreOrPostDatum { + PreDatum, + PostDatum, +} + +pub type ProgramDatumInclusions = HashMap; +pub type InclusionsFromConfig = HashMap; + +#[derive(Deserialize, Debug, Clone, Default)] +#[serde(rename_all = "camelCase")] +pub struct DatumInclusion { + #[serde(default)] + pub pre: bool, + #[serde(default)] + pub post: bool, + #[serde(default)] + pub length_exclusions: Vec, +} + +impl DatumInclusion { + pub fn can_include_datum(&self, pre_or_post: &PreOrPostDatum, data: &[u8]) -> bool { + let allow_pre_post = match pre_or_post { + PreOrPostDatum::PreDatum => self.pre, + PreOrPostDatum::PostDatum => self.post, + }; + + if !allow_pre_post { + return false; + } + + !self.length_exclusions.contains(&data.len()) + } +} + +#[derive(serde::Deserialize)] +struct GeyserConfigFileWithInclusions { + #[serde(rename = "datumProgramInclusions")] + pub datum_program_inclusions: InclusionsFromConfig, +} + +pub fn load_datum_program_inclusions(paths: &Option>) -> ProgramDatumInclusions { + let mut datum_program_inclusions: ProgramDatumInclusions = HashMap::new(); + if let Some(paths) = paths { + for path in paths { + let file = fs::read(path); + if !file.is_ok() { + eprintln!("Unable to read JSON file: {:?} Skipping...", path); + continue; + } + let json = serde_json::from_slice::(&file.unwrap()); + if let Err(e) = json { + eprintln!("Unable to parse JSON file {:?}: {:?} Skipping...", path, e); + continue; + } + let inclusions_map = json.unwrap().datum_program_inclusions; + for (pubkey, inc) in inclusions_map.iter() { + let pk_parsed = pubkey.parse::().expect( + format!( + "Bad pubkey provided to datumProgramInclusions in geyser config file {:?}", + path + ) + .as_str(), + ); + datum_program_inclusions.insert(pk_parsed, inc.clone()); + } + } + } + datum_program_inclusions +} diff --git a/runtime/src/runtime_config.rs b/runtime/src/runtime_config.rs index 2439dd85c2e46f..a2bdbff936e95d 100644 --- a/runtime/src/runtime_config.rs +++ b/runtime/src/runtime_config.rs @@ -1,9 +1,14 @@ +use std::sync::Arc; + use solana_program_runtime::compute_budget::ComputeBudget; +use crate::program_inclusions::ProgramDatumInclusions; + /// Encapsulates flags that can be used to tweak the runtime behavior. #[derive(AbiExample, Debug, Default, Clone)] pub struct RuntimeConfig { pub compute_budget: Option, pub log_messages_bytes_limit: Option, pub transaction_account_lock_limit: Option, + pub program_datum_inclusions: Arc, } diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 85c714c635b5b4..451cf70496b64b 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -248,6 +248,9 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, diff --git a/storage-proto/src/convert.rs b/storage-proto/src/convert.rs index e9070951942e2c..53d776bf09f00e 100644 --- a/storage-proto/src/convert.rs +++ b/storage-proto/src/convert.rs @@ -374,6 +374,7 @@ impl From for generated::TransactionStatusMeta { loaded_addresses, return_data, compute_units_consumed, + .. } = value; let err = match status { Ok(()) => None, @@ -527,6 +528,9 @@ impl TryFrom for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions, log_messages, pre_token_balances, diff --git a/storage-proto/src/lib.rs b/storage-proto/src/lib.rs index 0832690f1e2b88..09ccf0014e7f1e 100644 --- a/storage-proto/src/lib.rs +++ b/storage-proto/src/lib.rs @@ -200,6 +200,9 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions, log_messages, pre_token_balances: pre_token_balances @@ -231,6 +234,7 @@ impl TryFrom for StoredTransactionStatusMeta { loaded_addresses, return_data, compute_units_consumed, + .. } = value; if !loaded_addresses.is_empty() { diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index dddde5d7896b4e..f35583cc37606a 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -34,7 +34,8 @@ use { solana_rpc_client::{nonblocking, rpc_client::RpcClient}, solana_runtime::{ bank_forks::BankForks, genesis_utils::create_genesis_config_with_leader_ex, - runtime_config::RuntimeConfig, snapshot_config::SnapshotConfig, + program_inclusions::load_datum_program_inclusions, runtime_config::RuntimeConfig, + snapshot_config::SnapshotConfig, }, solana_sdk::{ account::{Account, AccountSharedData, WritableAccount}, @@ -984,6 +985,10 @@ impl TestValidator { ..AccountsDbConfig::default() }); + let program_datum_inclusions = Arc::new(load_datum_program_inclusions( + &config.geyser_plugin_config_files, + )); + let runtime_config = RuntimeConfig { compute_budget: config .compute_unit_limit @@ -993,6 +998,7 @@ impl TestValidator { }), log_messages_bytes_limit: config.log_messages_bytes_limit, transaction_account_lock_limit: config.transaction_account_lock_limit, + program_datum_inclusions, }; let mut validator_config = ValidatorConfig { diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index 0eb13d36819c4a..dd498352ab061a 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -353,6 +353,12 @@ pub struct TransactionStatusMeta { pub fee: u64, pub pre_balances: Vec, pub post_balances: Vec, + pub post_owners: Option>>, + //the first Option is for backward compat + //the inner Option is for if we filtered out the datum (exceeds max size) + //if account data is empty, that is "", not None + pub pre_datum: Option>>>, + pub post_datum: Option>>>, pub inner_instructions: Option>, pub log_messages: Option>, pub pre_token_balances: Option>, @@ -370,6 +376,9 @@ impl Default for TransactionStatusMeta { fee: 0, pre_balances: vec![], post_balances: vec![], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, @@ -391,6 +400,12 @@ pub struct UiTransactionStatusMeta { pub fee: u64, pub pre_balances: Vec, pub post_balances: Vec, + pub post_owners: Option>>, + //the first Option is for backward compat + //the inner Option is for if we filtered out the datum (exceeds max size) + //if account data is empty, that is "", not None + pub pre_datum: Option>>, + pub post_datum: Option>>, #[serde( default = "OptionSerializer::none", skip_serializing_if = "OptionSerializer::should_skip" @@ -467,6 +482,23 @@ impl UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: meta .inner_instructions .map(|ixs| { @@ -500,6 +532,23 @@ impl UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: OptionSerializer::Skip, log_messages: OptionSerializer::Skip, pre_token_balances: meta @@ -530,6 +579,23 @@ impl From for UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: meta .inner_instructions .map(|ixs| ixs.into_iter().map(Into::into).collect()) @@ -1054,7 +1120,7 @@ impl ConfirmedTransactionWithStatusMeta { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EncodedConfirmedTransactionWithStatusMeta { pub slot: Slot, @@ -1519,6 +1585,9 @@ mod test { fee: 1234, pre_balances: vec![1, 2, 3], post_balances: vec![4, 5, 6], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, diff --git a/validator/src/main.rs b/validator/src/main.rs index db65c890a7542e..d69d3e0c883496 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -54,6 +54,7 @@ use { solana_rpc_client::rpc_client::RpcClient, solana_rpc_client_api::config::RpcLeaderScheduleConfig, solana_runtime::{ + program_inclusions::load_datum_program_inclusions, runtime_config::RuntimeConfig, snapshot_bank_utils::DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, snapshot_config::{SnapshotConfig, SnapshotUsage}, @@ -1238,6 +1239,10 @@ pub fn main() { }; let starting_with_geyser_plugins: bool = on_start_geyser_plugin_config_files.is_some(); + let program_datum_inclusions = Arc::new(load_datum_program_inclusions( + &on_start_geyser_plugin_config_files, + )); + let rpc_bigtable_config = if matches.is_present("enable_rpc_bigtable_ledger_storage") || matches.is_present("enable_bigtable_ledger_upload") { @@ -1431,6 +1436,7 @@ pub fn main() { accounts_shrink_ratio, runtime_config: RuntimeConfig { log_messages_bytes_limit: value_of(&matches, "log_messages_bytes_limit"), + program_datum_inclusions, ..RuntimeConfig::default() }, staked_nodes_overrides: staked_nodes_overrides.clone(),