diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000000..f4bc0528e11b2d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,25 @@ +version: 2.1 + +orbs: + aws-ecr: circleci/aws-ecr@8.1.3 + +executors: + custom: + machine: + image: ubuntu-2204:2022.04.2 + resource_class: xlarge + +workflows: + run-pipeline: + jobs: + - aws-ecr/build-and-push-image: + executor: custom + dockerfile: _step-finance/Dockerfile + repo: step-solana + region: eu-north-1 + tag: ${CIRCLE_BRANCH},${CIRCLE_BRANCH}-<< pipeline.number >>,${CIRCLE_BRANCH}-<< pipeline.number >>-${CIRCLE_SHA1} + context: aws-account + filters: + branches: + only: + - step-release diff --git a/_step-finance/Dockerfile b/_step-finance/Dockerfile new file mode 100644 index 00000000000000..2edf8d4eb77ac3 --- /dev/null +++ b/_step-finance/Dockerfile @@ -0,0 +1,14 @@ +FROM rust:1.73-bullseye as builder +RUN apt-get update +RUN apt-get install -y libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang cmake make libprotobuf-dev protobuf-compiler +WORKDIR /solana-build +COPY . . +RUN cargo build --release + +FROM debian:bullseye-20230522 as runtime +RUN apt-get update +RUN apt-get install -y bzip2 +COPY --from=builder --chmod=700 /solana-build/target/release /usr/local/bin + +WORKDIR /solana +ENTRYPOINT ["solana-test-validator"] diff --git a/_step-finance/Dockerfile.dockerignore b/_step-finance/Dockerfile.dockerignore new file mode 100644 index 00000000000000..fdab7b85601aec --- /dev/null +++ b/_step-finance/Dockerfile.dockerignore @@ -0,0 +1,2 @@ +target/ +_step-finance/ diff --git a/cli/src/cluster_query.rs b/cli/src/cluster_query.rs index a5162e9498441a..1b4cc27597026f 100644 --- a/cli/src/cluster_query.rs +++ b/cli/src/cluster_query.rs @@ -2103,6 +2103,7 @@ pub fn process_transaction_history( block_time, slot, transaction: transaction_with_meta, + index_in_block: _, } = confirmed_transaction; let decoded_transaction = diff --git a/cli/src/wallet.rs b/cli/src/wallet.rs index 04b891562f395e..6121103c6d8796 100644 --- a/cli/src/wallet.rs +++ b/cli/src/wallet.rs @@ -776,6 +776,7 @@ pub fn process_confirm( block_time, slot, transaction: transaction_with_meta, + index_in_block: _, } = confirmed_transaction; let decoded_transaction = diff --git a/core/src/banking_stage/committer.rs b/core/src/banking_stage/committer.rs index ab8f3a9ed57b5b..86cd5825cbda55 100644 --- a/core/src/banking_stage/committer.rs +++ b/core/src/banking_stage/committer.rs @@ -10,9 +10,10 @@ use { }, solana_measure::measure_us, solana_runtime::{ - bank::{Bank, CommitTransactionCounts, TransactionBalancesSet}, + bank::{Bank, CommitTransactionCounts, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet}, bank_utils, prioritization_fee_cache::PrioritizationFeeCache, + program_inclusions::PreOrPostDatum, transaction_batch::TransactionBatch, }, solana_sdk::{hash::Hash, pubkey::Pubkey, saturating_add_assign}, @@ -32,6 +33,7 @@ pub enum CommitTransactionDetails { #[derive(Default)] pub(super) struct PreBalanceInfo { pub native: Vec>, + pub datum: Vec>>>, pub token: Vec>, pub mint_decimals: HashMap, } @@ -142,7 +144,8 @@ impl Committer { ) { if let Some(transaction_status_sender) = &self.transaction_status_sender { let txs = batch.sanitized_transactions().to_vec(); - let post_balances = bank.collect_balances(batch); + let (post_balances, post_datum, owners) = + bank.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum); let post_token_balances = collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals); let mut transaction_index = starting_transaction_index.unwrap_or_default(); @@ -167,6 +170,10 @@ impl Committer { std::mem::take(&mut pre_balance_info.native), post_balances, ), + TransactionOwnersSet { + owners, + }, + TransactionDatumSet::new(std::mem::take(&mut pre_balance_info.datum), post_datum), TransactionTokenBalancesSet::new( std::mem::take(&mut pre_balance_info.token), post_token_balances, diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 1b8487a923b972..c716b140789e2d 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -25,6 +25,7 @@ use { solana_runtime::{ accounts::validate_fee_payer, bank::{Bank, LoadAndExecuteTransactionsOutput}, + program_inclusions::PreOrPostDatum, transaction_batch::TransactionBatch, }, solana_sdk::{ @@ -570,7 +571,8 @@ impl Consumer { // If the extra meta-data services are enabled for RPC, collect the // pre-balances for native and token programs. if transaction_status_sender_enabled { - pre_balance_info.native = bank.collect_balances(batch); + (pre_balance_info.native, pre_balance_info.datum, ..) = + bank.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum); pre_balance_info.token = collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals) } diff --git a/core/src/validator.rs b/core/src/validator.rs index 6d2a9fb47af9ca..d177d8d77ba629 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -512,6 +512,7 @@ impl Validator { let mut bank_notification_senders = Vec::new(); let exit = Arc::new(AtomicBool::new(false)); + let inclusions_copy = config.runtime_config.program_datum_inclusions.clone(); let geyser_plugin_service = if let Some(geyser_plugin_config_files) = &config.on_start_geyser_plugin_config_files { @@ -524,6 +525,7 @@ impl Validator { confirmed_bank_receiver, geyser_plugin_config_files, rpc_to_plugin_manager_receiver_and_exit, + inclusions_copy, ) .map_err(|err| format!("Failed to load the Geyser plugin: {err:?}"))?, ) diff --git a/geyser-plugin-manager/src/geyser_plugin_manager.rs b/geyser-plugin-manager/src/geyser_plugin_manager.rs index 02792525ad370c..f23d26a2025371 100644 --- a/geyser-plugin-manager/src/geyser_plugin_manager.rs +++ b/geyser-plugin-manager/src/geyser_plugin_manager.rs @@ -4,9 +4,11 @@ use { libloading::Library, log::*, solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, + solana_runtime::program_inclusions::{load_datum_program_inclusions, ProgramDatumInclusions}, std::{ ops::{Deref, DerefMut}, path::Path, + sync::{Arc, RwLock}, }, }; @@ -47,13 +49,15 @@ impl DerefMut for LoadedGeyserPlugin { pub struct GeyserPluginManager { pub plugins: Vec, libs: Vec, + inclusions: Arc>, } impl GeyserPluginManager { - pub fn new() -> Self { + pub fn new(inclusions: Arc>) -> Self { GeyserPluginManager { plugins: Vec::default(), libs: Vec::default(), + inclusions, } } @@ -239,6 +243,13 @@ impl GeyserPluginManager { Ok(()) => { self.plugins.push(new_plugin); self.libs.push(new_lib); + + // Reload datum inclusions + log::info!("Reloading datum inclusions"); + let mut inclusions_write_lock = self.inclusions.write().unwrap(); + *inclusions_write_lock = + load_datum_program_inclusions(&Some(vec![config_file.into()])); + log::info!("Reloaded datum inclusions"); } // On failure, return error @@ -483,7 +494,9 @@ mod tests { #[test] fn test_geyser_reload() { // Initialize empty manager - let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new())); + let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new( + RwLock::new(Default::default()), + )))); // No plugins are loaded, this should fail let mut plugin_manager_lock = plugin_manager.write().unwrap(); @@ -524,7 +537,9 @@ mod tests { #[test] fn test_plugin_list() { // Initialize empty manager - let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new())); + let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new( + RwLock::new(Default::default()), + )))); let mut plugin_manager_lock = plugin_manager.write().unwrap(); // Load two plugins @@ -548,7 +563,9 @@ mod tests { #[test] fn test_plugin_load_unload() { // Initialize empty manager - let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new())); + let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new( + RwLock::new(Default::default()), + )))); let mut plugin_manager_lock = plugin_manager.write().unwrap(); // Load rpc call diff --git a/geyser-plugin-manager/src/geyser_plugin_service.rs b/geyser-plugin-manager/src/geyser_plugin_service.rs index ff3e050dc4b391..52a9bb03101d7d 100644 --- a/geyser-plugin-manager/src/geyser_plugin_service.rs +++ b/geyser-plugin-manager/src/geyser_plugin_service.rs @@ -17,6 +17,7 @@ use { optimistically_confirmed_bank_tracker::SlotNotification, transaction_notifier_interface::TransactionNotifierArc, }, + solana_runtime::program_inclusions::ProgramDatumInclusions, std::{ path::{Path, PathBuf}, sync::{ @@ -55,8 +56,14 @@ impl GeyserPluginService { pub fn new( confirmed_bank_receiver: Receiver, geyser_plugin_config_files: &[PathBuf], + inclusions: Arc>, ) -> Result { - Self::new_with_receiver(confirmed_bank_receiver, geyser_plugin_config_files, None) + Self::new_with_receiver( + confirmed_bank_receiver, + geyser_plugin_config_files, + None, + inclusions, + ) } pub fn new_with_receiver( @@ -66,12 +73,13 @@ impl GeyserPluginService { Receiver, Arc, )>, + inclusions: Arc>, ) -> Result { info!( "Starting GeyserPluginService from config files: {:?}", geyser_plugin_config_files ); - let mut plugin_manager = GeyserPluginManager::new(); + let mut plugin_manager = GeyserPluginManager::new(inclusions); for geyser_plugin_config_file in geyser_plugin_config_files { Self::load_plugin(&mut plugin_manager, geyser_plugin_config_file)?; diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index eaf2f2277cd599..cc2373c7dabfbc 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -259,9 +259,16 @@ pub fn load_and_process_ledger( let (confirmed_bank_sender, confirmed_bank_receiver) = unbounded(); drop(confirmed_bank_sender); - let geyser_service = - GeyserPluginService::new(confirmed_bank_receiver, &geyser_config_files) - .map_err(LoadAndProcessLedgerError::GeyserServiceSetup)?; + let inclusions_copy = process_options + .runtime_config + .program_datum_inclusions + .clone(); + let geyser_service = GeyserPluginService::new( + confirmed_bank_receiver, + &geyser_config_files, + inclusions_copy, + ) + .map_err(LoadAndProcessLedgerError::GeyserServiceSetup)?; ( geyser_service.get_accounts_update_notifier(), geyser_service.get_transaction_notifier(), diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 4e093814b9bb3a..2749d5ad66d4c2 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -35,7 +35,7 @@ use { solana_rayon_threadlimit::{get_max_thread_count, get_thread_count}, solana_runtime::{ accounts_background_service::{AbsRequestSender, SnapshotRequestKind}, - bank::{Bank, TransactionBalancesSet}, + bank::{Bank, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet}, bank_forks::BankForks, bank_utils, commitment::VOTE_THRESHOLD_SIZE, @@ -157,7 +157,7 @@ pub fn execute_batch( vec![] }; - let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions( + let (tx_results, balances, datum, owners) = batch.bank().load_execute_and_commit_transactions( batch, MAX_PROCESSING_AGE, transaction_status_sender.is_some(), @@ -203,6 +203,8 @@ pub fn execute_batch( transactions, execution_results, balances, + owners, + datum, token_balances, rent_debits, transaction_indexes.to_vec(), @@ -1828,6 +1830,8 @@ pub struct TransactionStatusBatch { pub transactions: Vec, pub execution_results: Vec>, pub balances: TransactionBalancesSet, + pub owners: TransactionOwnersSet, + pub datum: TransactionDatumSet, pub token_balances: TransactionTokenBalancesSet, pub rent_debits: Vec, pub transaction_indexes: Vec, @@ -1845,6 +1849,8 @@ impl TransactionStatusSender { transactions: Vec, execution_results: Vec, balances: TransactionBalancesSet, + owners: TransactionOwnersSet, + datum: TransactionDatumSet, token_balances: TransactionTokenBalancesSet, rent_debits: Vec, transaction_indexes: Vec, @@ -1864,6 +1870,8 @@ impl TransactionStatusSender { }) .collect(), balances, + owners, + datum, token_balances, rent_debits, transaction_indexes, @@ -3951,6 +3959,8 @@ pub mod tests { .. }, _balances, + _datums, + _owners, ) = batch.bank().load_execute_and_commit_transactions( &batch, MAX_PROCESSING_AGE, diff --git a/rpc-client/src/mock_sender.rs b/rpc-client/src/mock_sender.rs index 44ab26359c3f32..6c364bd0609a27 100644 --- a/rpc-client/src/mock_sender.rs +++ b/rpc-client/src/mock_sender.rs @@ -226,6 +226,9 @@ impl RpcSender for MockSender { fee: 0, pre_balances: vec![499999999999999950, 50, 1], post_balances: vec![499999999999999950, 50, 1], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: OptionSerializer::None, log_messages: OptionSerializer::None, pre_token_balances: OptionSerializer::None, @@ -237,6 +240,7 @@ impl RpcSender for MockSender { }), }, block_time: Some(1628633791), + index_in_block: 0, })?, "getTransactionCount" => json![1234], "getSlot" => json![0], diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index e007ad95975798..0ebcbb78e464da 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -1502,7 +1502,7 @@ impl JsonRpcRequestProcessor { let encode_transaction = |confirmed_tx_with_meta: ConfirmedTransactionWithStatusMeta| -> Result { - Ok(confirmed_tx_with_meta.encode(encoding, max_supported_transaction_version).map_err(RpcCustomError::from)?) + Ok(confirmed_tx_with_meta.encode(encoding, max_supported_transaction_version, 0).map_err(RpcCustomError::from)?) }; match confirmed_transaction.unwrap_or(None) { diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index eca53c66658766..28b5462ebd103d 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -69,6 +69,8 @@ impl TransactionStatusService { bank, transactions, execution_results, + owners, + datum, balances, token_balances, rent_debits, @@ -80,6 +82,9 @@ impl TransactionStatusService { execution_result, pre_balances, post_balances, + owners, + pre_datum, + post_datum, pre_token_balances, post_token_balances, rent_debits, @@ -89,6 +94,9 @@ impl TransactionStatusService { execution_results, balances.pre_balances, balances.post_balances, + owners.owners, + datum.pre_datum, + datum.post_datum, token_balances.pre_token_balances, token_balances.post_token_balances, rent_debits, @@ -144,6 +152,9 @@ impl TransactionStatusService { fee, pre_balances, post_balances, + post_owners: Some(owners), + pre_datum: Some(pre_datum), + post_datum: Some(post_datum), inner_instructions, log_messages, pre_token_balances, @@ -206,6 +217,7 @@ impl TransactionStatusService { #[cfg(test)] pub(crate) mod tests { + use { super::*, crate::transaction_notifier_interface::TransactionNotifier, @@ -217,7 +229,9 @@ pub(crate) mod tests { rent_debits::RentDebits, }, solana_ledger::{genesis_utils::create_genesis_config, get_tmp_ledger_path_auto_delete}, - solana_runtime::bank::{Bank, TransactionBalancesSet}, + solana_runtime::bank::{ + Bank, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet, + }, solana_sdk::{ account_utils::StateMut, clock::Slot, @@ -410,6 +424,11 @@ pub(crate) mod tests { let transaction_index: usize = bank.transaction_count().try_into().unwrap(); let transaction_status_batch = TransactionStatusBatch { bank, + owners: TransactionOwnersSet { owners: vec![] }, + datum: TransactionDatumSet { + post_datum: vec![vec![Some(vec![0x69])]], + pre_datum: vec![vec![Some(vec![0x04, 0x20])]], + }, transactions: vec![transaction], execution_results: vec![transaction_result], balances, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 8b217d3b8cf679..c22814735fcade 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -48,6 +48,7 @@ use { epoch_rewards_hasher::hash_rewards_into_partitions, epoch_stakes::{EpochStakes, NodeVoteAccounts}, installed_scheduler_pool::{BankWithScheduler, InstalledSchedulerRwLock}, + program_inclusions::{PreOrPostDatum, ProgramDatumInclusions}, runtime_config::RuntimeConfig, serde_snapshot::BankIncrementalSnapshotPersistence, snapshot_hash::SnapshotHash, @@ -355,6 +356,27 @@ impl TransactionBalancesSet { } pub type TransactionBalances = Vec>; +pub struct TransactionDatumSet { + pub pre_datum: TransactionDatum, + pub post_datum: TransactionDatum, +} + +impl TransactionDatumSet { + pub fn new(pre_datum: TransactionDatum, post_datum: TransactionDatum) -> Self { + Self { + pre_datum, + post_datum, + } + } +} +pub type TransactionDatum = Vec>>>; + +pub struct TransactionOwnersSet { + pub owners: TransactionOwners, +} + +pub type TransactionOwners = Vec>>; + /// A list of log messages emitted during a transaction pub type TransactionLogMessages = Vec; @@ -570,6 +592,7 @@ impl PartialEq for Bank { loaded_programs_cache: _, check_program_modification_slot: _, epoch_reward_status: _, + program_datum_inclusions: _, // Ignore new fields explicitly if they do not impact PartialEq. // Adding ".." will remove compile-time checks that if a new field // is added to the struct, this PartialEq is accordingly updated. @@ -833,6 +856,8 @@ pub struct Bank { pub check_program_modification_slot: bool, epoch_reward_status: EpochRewardStatus, + /// programs that will have their account data sent to geyser + pub program_datum_inclusions: Arc>, } struct VoteWithStakeDelegations { @@ -1020,6 +1045,7 @@ impl Bank { ))), check_program_modification_slot: false, epoch_reward_status: EpochRewardStatus::default(), + program_datum_inclusions: Arc::new(RwLock::new(ProgramDatumInclusions::default())), }; let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64; @@ -1054,6 +1080,7 @@ impl Bank { ); let accounts = Accounts::new(Arc::new(accounts_db)); let mut bank = Self::default_with_accounts(accounts); + bank.program_datum_inclusions = runtime_config.program_datum_inclusions.clone(); bank.ancestors = Ancestors::from(vec![bank.slot()]); bank.transaction_debug_keys = debug_keys; bank.runtime_config = runtime_config; @@ -1331,6 +1358,7 @@ impl Bank { loaded_programs_cache: parent.loaded_programs_cache.clone(), check_program_modification_slot: false, epoch_reward_status: parent.epoch_reward_status.clone(), + program_datum_inclusions: parent.program_datum_inclusions.clone(), }; let (_, ancestors_time_us) = measure_us!({ @@ -1767,6 +1795,7 @@ impl Bank { a corrupted snapshot or bugs in cached accounts or accounts-db.", ); let stakes_accounts_load_duration = now.elapsed(); + let program_datum_inclusions = runtime_config.program_datum_inclusions.clone(); let mut bank = Self { skipped_rewrites: Mutex::default(), incremental_snapshot_persistence: fields.incremental_snapshot_persistence, @@ -1833,6 +1862,7 @@ impl Bank { ))), check_program_modification_slot: false, epoch_reward_status: fields.epoch_reward_status, + program_datum_inclusions, }; bank.finish_init( genesis_config, @@ -4532,6 +4562,31 @@ impl Bank { balances } + pub fn collect_balances_and_datum( + &self, + batch: &TransactionBatch, + pre_or_post: PreOrPostDatum, + ) -> (TransactionBalances, TransactionDatum, TransactionOwners) { + let mut balances: TransactionBalances = vec![]; + let mut datum: TransactionDatum = vec![]; + let mut owners: TransactionOwners = vec![]; + for transaction in batch.sanitized_transactions() { + let mut transaction_balances: Vec = vec![]; + let mut transaction_datum: Vec>> = vec![]; + let mut transaction_owners: Vec> = vec![]; + for account_key in transaction.message().account_keys().iter() { + let (balance, data, owner) = self.get_balance_and_data(account_key, &pre_or_post); + transaction_balances.push(balance); + transaction_datum.push(data); + transaction_owners.push(owner); + } + balances.push(transaction_balances); + datum.push(transaction_datum); + owners.push(transaction_owners); + } + (balances, datum, owners) + } + fn program_modification_slot(&self, pubkey: &Pubkey) -> Result { let program = self .get_account_with_fixed_root(pubkey) @@ -6293,11 +6348,16 @@ impl Bank { enable_return_data_recording: bool, timings: &mut ExecuteTimings, log_messages_bytes_limit: Option, - ) -> (TransactionResults, TransactionBalancesSet) { - let pre_balances = if collect_balances { - self.collect_balances(batch) + ) -> ( + TransactionResults, + TransactionBalancesSet, + TransactionDatumSet, + TransactionOwnersSet, + ) { + let (pre_balances, pre_datum, ..) = if collect_balances { + self.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum) } else { - vec![] + (vec![], vec![], vec![]) }; let LoadAndExecuteTransactionsOutput { @@ -6338,14 +6398,16 @@ impl Bank { }, timings, ); - let post_balances = if collect_balances { - self.collect_balances(batch) + let (post_balances, post_datum, owners) = if collect_balances { + self.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum) } else { - vec![] + (vec![], vec![], vec![]) }; ( results, TransactionBalancesSet::new(pre_balances, post_balances), + TransactionDatumSet::new(pre_datum, post_datum), + TransactionOwnersSet { owners }, ) } @@ -6441,6 +6503,24 @@ impl Bank { pub fn read_balance(account: &AccountSharedData) -> u64 { account.lamports() } + + pub fn read_data( + &self, + account: &AccountSharedData, + pre_or_post: &PreOrPostDatum, + ) -> Option> { + let data = account.data(); + let owner = account.owner(); + let inclusions_lock = self.program_datum_inclusions.read().unwrap(); + let inclusion = inclusions_lock.get(owner)?; + let include_data = inclusion.can_include_datum(pre_or_post, &data); + + if !include_data { + None + } else { + Some(data.to_vec()) + } + } /// Each program would need to be able to introspect its own state /// this is hard-coded to the Budget language pub fn get_balance(&self, pubkey: &Pubkey) -> u64 { @@ -6448,6 +6528,21 @@ impl Bank { .map(|x| Self::read_balance(&x)) .unwrap_or(0) } + pub fn get_balance_and_data( + &self, + pubkey: &Pubkey, + pre_or_post: &PreOrPostDatum, + ) -> (u64, Option>, Option) { + self.get_account(pubkey) + .map(|x| { + ( + Self::read_balance(&x), + Self::read_data(self, &x, pre_or_post), + Some(*x.owner()), + ) + }) + .unwrap_or((0, None, None)) + } /// Compute all the parents of the bank in order pub fn parents(&self) -> Vec> { diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 18c9727a991d73..ae5ca7f8bf6456 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -5941,7 +5941,7 @@ fn test_pre_post_transaction_balances() { let txs = vec![tx0, tx1, tx2]; let lock_result = bank0.prepare_batch_for_tests(txs); - let (transaction_results, transaction_balances_set) = bank0 + let (transaction_results, transaction_balances_set, _) = bank0 .load_execute_and_commit_transactions( &lock_result, MAX_PROCESSING_AGE, diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index b0884a6f185c20..93b5b155d5b6e9 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -21,6 +21,7 @@ pub mod loader_utils; pub mod non_circulating_supply; pub mod prioritization_fee; pub mod prioritization_fee_cache; +pub mod program_inclusions; pub mod root_bank_cache; pub mod runtime_config; pub mod serde_snapshot; diff --git a/runtime/src/program_inclusions.rs b/runtime/src/program_inclusions.rs new file mode 100644 index 00000000000000..e4c6c21bdee66d --- /dev/null +++ b/runtime/src/program_inclusions.rs @@ -0,0 +1,73 @@ +use std::{collections::HashMap, fs, path::PathBuf}; + +use solana_sdk::pubkey::Pubkey; + +pub enum PreOrPostDatum { + PreDatum, + PostDatum, +} + +pub type ProgramDatumInclusions = HashMap; +pub type InclusionsFromConfig = HashMap; + +#[derive(Deserialize, Debug, Clone, Default)] +#[serde(rename_all = "camelCase")] +pub struct DatumInclusion { + #[serde(default)] + pub pre: bool, + #[serde(default)] + pub post: bool, + #[serde(default)] + pub length_exclusions: Vec, +} + +impl DatumInclusion { + pub fn can_include_datum(&self, pre_or_post: &PreOrPostDatum, data: &[u8]) -> bool { + let allow_pre_post = match pre_or_post { + PreOrPostDatum::PreDatum => self.pre, + PreOrPostDatum::PostDatum => self.post, + }; + + if !allow_pre_post { + return false; + } + + !self.length_exclusions.contains(&data.len()) + } +} + +#[derive(serde::Deserialize)] +struct GeyserConfigFileWithInclusions { + #[serde(rename = "datumProgramInclusions")] + pub datum_program_inclusions: InclusionsFromConfig, +} + +pub fn load_datum_program_inclusions(paths: &Option>) -> ProgramDatumInclusions { + let mut datum_program_inclusions: ProgramDatumInclusions = HashMap::new(); + if let Some(paths) = paths { + for path in paths { + let file = fs::read(path); + if !file.is_ok() { + eprintln!("Unable to read JSON file: {:?} Skipping...", path); + continue; + } + let json = serde_json::from_slice::(&file.unwrap()); + if let Err(e) = json { + eprintln!("Unable to parse JSON file {:?}: {:?} Skipping...", path, e); + continue; + } + let inclusions_map = json.unwrap().datum_program_inclusions; + for (pubkey, inc) in inclusions_map.iter() { + let pk_parsed = pubkey.parse::().expect( + format!( + "Bad pubkey provided to datumProgramInclusions in geyser config file {:?}", + path + ) + .as_str(), + ); + datum_program_inclusions.insert(pk_parsed, inc.clone()); + } + } + } + datum_program_inclusions +} diff --git a/runtime/src/runtime_config.rs b/runtime/src/runtime_config.rs index 2439dd85c2e46f..4b4dfef754741c 100644 --- a/runtime/src/runtime_config.rs +++ b/runtime/src/runtime_config.rs @@ -1,9 +1,14 @@ +use std::sync::{Arc, RwLock}; + use solana_program_runtime::compute_budget::ComputeBudget; +use crate::program_inclusions::ProgramDatumInclusions; + /// Encapsulates flags that can be used to tweak the runtime behavior. #[derive(AbiExample, Debug, Default, Clone)] pub struct RuntimeConfig { pub compute_budget: Option, pub log_messages_bytes_limit: Option, pub transaction_account_lock_limit: Option, + pub program_datum_inclusions: Arc>, } diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 85c714c635b5b4..451cf70496b64b 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -248,6 +248,9 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, diff --git a/storage-proto/src/convert.rs b/storage-proto/src/convert.rs index e9070951942e2c..53d776bf09f00e 100644 --- a/storage-proto/src/convert.rs +++ b/storage-proto/src/convert.rs @@ -374,6 +374,7 @@ impl From for generated::TransactionStatusMeta { loaded_addresses, return_data, compute_units_consumed, + .. } = value; let err = match status { Ok(()) => None, @@ -527,6 +528,9 @@ impl TryFrom for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions, log_messages, pre_token_balances, diff --git a/storage-proto/src/lib.rs b/storage-proto/src/lib.rs index 0832690f1e2b88..09ccf0014e7f1e 100644 --- a/storage-proto/src/lib.rs +++ b/storage-proto/src/lib.rs @@ -200,6 +200,9 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions, log_messages, pre_token_balances: pre_token_balances @@ -231,6 +234,7 @@ impl TryFrom for StoredTransactionStatusMeta { loaded_addresses, return_data, compute_units_consumed, + .. } = value; if !loaded_addresses.is_empty() { diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index dddde5d7896b4e..823e54e4e0a404 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -34,7 +34,8 @@ use { solana_rpc_client::{nonblocking, rpc_client::RpcClient}, solana_runtime::{ bank_forks::BankForks, genesis_utils::create_genesis_config_with_leader_ex, - runtime_config::RuntimeConfig, snapshot_config::SnapshotConfig, + program_inclusions::load_datum_program_inclusions, runtime_config::RuntimeConfig, + snapshot_config::SnapshotConfig, }, solana_sdk::{ account::{Account, AccountSharedData, WritableAccount}, @@ -179,7 +180,9 @@ impl Default for TestValidatorGenesis { log_messages_bytes_limit: Option::::default(), transaction_account_lock_limit: Option::::default(), tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP, - geyser_plugin_manager: Arc::new(RwLock::new(GeyserPluginManager::new())), + geyser_plugin_manager: Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new( + RwLock::new(Default::default()), + )))), admin_rpc_service_post_init: Arc::>>::default(), } @@ -984,6 +987,10 @@ impl TestValidator { ..AccountsDbConfig::default() }); + let program_datum_inclusions = Arc::new(RwLock::new(load_datum_program_inclusions( + &config.geyser_plugin_config_files, + ))); + let runtime_config = RuntimeConfig { compute_budget: config .compute_unit_limit @@ -993,6 +1000,7 @@ impl TestValidator { }), log_messages_bytes_limit: config.log_messages_bytes_limit, transaction_account_lock_limit: config.transaction_account_lock_limit, + program_datum_inclusions, }; let mut validator_config = ValidatorConfig { diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index 0eb13d36819c4a..497b6adcb52178 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -353,6 +353,12 @@ pub struct TransactionStatusMeta { pub fee: u64, pub pre_balances: Vec, pub post_balances: Vec, + pub post_owners: Option>>, + //the first Option is for backward compat + //the inner Option is for if we filtered out the datum (exceeds max size) + //if account data is empty, that is "", not None + pub pre_datum: Option>>>, + pub post_datum: Option>>>, pub inner_instructions: Option>, pub log_messages: Option>, pub pre_token_balances: Option>, @@ -370,6 +376,9 @@ impl Default for TransactionStatusMeta { fee: 0, pre_balances: vec![], post_balances: vec![], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, @@ -391,6 +400,12 @@ pub struct UiTransactionStatusMeta { pub fee: u64, pub pre_balances: Vec, pub post_balances: Vec, + pub post_owners: Option>>, + //the first Option is for backward compat + //the inner Option is for if we filtered out the datum (exceeds max size) + //if account data is empty, that is "", not None + pub pre_datum: Option>>, + pub post_datum: Option>>, #[serde( default = "OptionSerializer::none", skip_serializing_if = "OptionSerializer::should_skip" @@ -467,6 +482,23 @@ impl UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: meta .inner_instructions .map(|ixs| { @@ -500,6 +532,23 @@ impl UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: OptionSerializer::Skip, log_messages: OptionSerializer::Skip, pre_token_balances: meta @@ -530,6 +579,23 @@ impl From for UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: meta .inner_instructions .map(|ixs| ixs.into_iter().map(Into::into).collect()) @@ -1037,6 +1103,7 @@ impl ConfirmedTransactionWithStatusMeta { self, encoding: UiTransactionEncoding, max_supported_transaction_version: Option, + index_in_block: usize, ) -> Result { Ok(EncodedConfirmedTransactionWithStatusMeta { slot: self.slot, @@ -1046,6 +1113,7 @@ impl ConfirmedTransactionWithStatusMeta { true, )?, block_time: self.block_time, + index_in_block, }) } @@ -1054,13 +1122,14 @@ impl ConfirmedTransactionWithStatusMeta { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EncodedConfirmedTransactionWithStatusMeta { pub slot: Slot, #[serde(flatten)] pub transaction: EncodedTransactionWithStatusMeta, pub block_time: Option, + pub index_in_block: usize, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1519,6 +1588,9 @@ mod test { fee: 1234, pre_balances: vec![1, 2, 3], post_balances: vec![4, 5, 6], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, diff --git a/validator/src/main.rs b/validator/src/main.rs index acc77c16cb3d5c..38799e0bd16fb3 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -45,6 +45,7 @@ use { solana_rpc_client::rpc_client::RpcClient, solana_rpc_client_api::config::RpcLeaderScheduleConfig, solana_runtime::{ + program_inclusions::load_datum_program_inclusions, runtime_config::RuntimeConfig, snapshot_bank_utils::DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, snapshot_config::{SnapshotConfig, SnapshotUsage}, @@ -1238,6 +1239,10 @@ pub fn main() { }; let starting_with_geyser_plugins: bool = on_start_geyser_plugin_config_files.is_some(); + let program_datum_inclusions = Arc::new(RwLock::new(load_datum_program_inclusions( + &on_start_geyser_plugin_config_files, + ))); + let rpc_bigtable_config = if matches.is_present("enable_rpc_bigtable_ledger_storage") || matches.is_present("enable_bigtable_ledger_upload") { @@ -1431,6 +1436,7 @@ pub fn main() { accounts_shrink_ratio, runtime_config: RuntimeConfig { log_messages_bytes_limit: value_of(&matches, "log_messages_bytes_limit"), + program_datum_inclusions, ..RuntimeConfig::default() }, staked_nodes_overrides: staked_nodes_overrides.clone(),