From 1f774e751d635532a22991b9142f43a57d8f3a9e Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Tue, 17 Dec 2024 00:42:47 +0900 Subject: [PATCH 01/17] add/change some comments --- stats/stats/src/data_source/kinds/remote_db/mod.rs | 6 +++--- stats/stats/src/range.rs | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/stats/stats/src/data_source/kinds/remote_db/mod.rs b/stats/stats/src/data_source/kinds/remote_db/mod.rs index 5a47180d4..6d54909e3 100644 --- a/stats/stats/src/data_source/kinds/remote_db/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/mod.rs @@ -2,10 +2,10 @@ //! //! The main application - SQL queries from remote (=blockscout) database. //! -//! ## Details +//! ## Note //! -//! This source does not have any persistency and is only an adapter for representing -//! a remote DB as a `DataSource`. +//! [`RemoteDatabaseSource`] usually does not have any persistency and is only an +//! adapter for representing a remote DB as a `DataSource`. //! //! Since each [`RemoteQueryBehaviour::query_data`] performs (likely a heavy) database //! query, it is undesireable to have this source present in more than one place. diff --git a/stats/stats/src/range.rs b/stats/stats/src/range.rs index 56b62790b..65c1adb5a 100644 --- a/stats/stats/src/range.rs +++ b/stats/stats/src/range.rs @@ -192,6 +192,9 @@ pub fn exclusive_range_to_inclusive( start..=end } +/// Note: databases usually don't operate with sub-millisecond precision (as does +/// [`chrono::DateTime`], for example), so the result of this conversion might +/// not be actually accurate, as the endpoint will get excluded on query. pub fn inclusive_range_to_exclusive(r: RangeInclusive) -> Range { let (start, end) = r.into_inner(); // impossible to include max value in exclusive range, @@ -305,6 +308,7 @@ where }; Ok(range) } + #[cfg(test)] mod tests { use super::*; From 74229a0db361b2788ac94190919c8c237ae20815 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Tue, 17 Dec 2024 00:42:56 +0900 Subject: [PATCH 02/17] add charts (straightforward) --- .../charts/counters/average_txn_fee_24h.rs | 183 ++++++++++++++++++ stats/stats/src/charts/counters/mod.rs | 4 + .../stats/src/charts/counters/new_txns_24h.rs | 147 ++++++++++++++ .../stats/src/charts/counters/pending_txns.rs | 105 ++++++++++ .../stats/src/charts/counters/txns_fee_24h.rs | 173 +++++++++++++++++ 5 files changed, 612 insertions(+) create mode 100644 stats/stats/src/charts/counters/average_txn_fee_24h.rs create mode 100644 stats/stats/src/charts/counters/new_txns_24h.rs create mode 100644 stats/stats/src/charts/counters/pending_txns.rs create mode 100644 stats/stats/src/charts/counters/txns_fee_24h.rs diff --git a/stats/stats/src/charts/counters/average_txn_fee_24h.rs b/stats/stats/src/charts/counters/average_txn_fee_24h.rs new file mode 100644 index 000000000..eef6c97d5 --- /dev/null +++ b/stats/stats/src/charts/counters/average_txn_fee_24h.rs @@ -0,0 +1,183 @@ +use std::ops::Range; + +use crate::{ + data_source::{ + kinds::{ + data_manipulation::map::MapToString, + local_db::DirectPointLocalDbChartSource, + remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, + }, + types::BlockscoutMigrations, + UpdateContext, + }, + range::{inclusive_range_to_exclusive, UniversalRange}, + types::TimespanValue, + utils::{produce_filter_and_values, sql_with_range_filter_opt}, + ChartError, ChartProperties, MissingDatePolicy, Named, +}; +use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; +use entity::sea_orm_active_enums::ChartType; +use sea_orm::{DbBackend, FromQueryResult, Statement}; + +const ETHER: i64 = i64::pow(10, 18); + +/// `AverageTxnFeeStatement` but without `group by` +pub struct AverageTxnFeeUngroupedStatement; + +impl StatementFromRange for AverageTxnFeeUngroupedStatement { + fn get_statement( + range: Option>>, + completed_migrations: &BlockscoutMigrations, + ) -> Statement { + if completed_migrations.denormalization { + // TODO: consider supporting such case in macro ? + let mut args = vec![ETHER.into()]; + let (tx_filter, new_args) = + produce_filter_and_values(range.clone(), "t.block_timestamp", args.len() + 1); + args.extend(new_args); + let (block_filter, new_args) = + produce_filter_and_values(range.clone(), "b.timestamp", args.len() + 1); + args.extend(new_args); + let sql = format!( + r#" + SELECT + (AVG( + t_filtered.gas_used * + COALESCE( + t_filtered.gas_price, + b.base_fee_per_gas + LEAST( + t_filtered.max_priority_fee_per_gas, + t_filtered.max_fee_per_gas - b.base_fee_per_gas + ) + ) + ) / $1)::FLOAT as value + FROM ( + SELECT * from transactions t + WHERE + t.block_consensus = true AND + t.block_timestamp != to_timestamp(0) {tx_filter} + ) as t_filtered + JOIN blocks b ON t_filtered.block_hash = b.hash + WHERE + b.timestamp != to_timestamp(0) AND + b.consensus = true {block_filter} + "#, + ); + Statement::from_sql_and_values(DbBackend::Postgres, sql, args) + } else { + sql_with_range_filter_opt!( + DbBackend::Postgres, + r#" + SELECT + (AVG( + t.gas_used * + COALESCE( + t.gas_price, + b.base_fee_per_gas + LEAST( + t.max_priority_fee_per_gas, + t.max_fee_per_gas - b.base_fee_per_gas + ) + ) + ) / $1)::FLOAT as value + FROM transactions t + JOIN blocks b ON t.block_hash = b.hash + WHERE + b.timestamp != to_timestamp(0) AND + b.consensus = true {filter} + "#, + [ETHER.into()], + "b.timestamp", + range + ) + } + } +} + +#[derive(FromQueryResult)] +struct Value { + value: f64, +} + +pub struct AverageTxnFee24hQuery; + +impl RemoteQueryBehaviour for AverageTxnFee24hQuery { + type Output = TimespanValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result { + let update_time = cx.time; + let range_24h = update_time + .checked_sub_signed(TimeDelta::hours(24)) + .unwrap_or(DateTime::::MIN_UTC)..=update_time; + let query = AverageTxnFeeUngroupedStatement::get_statement( + Some(inclusive_range_to_exclusive(range_24h)), + &cx.blockscout_applied_migrations, + ); + + let data = Value::find_by_statement(query) + .one(cx.blockscout.connection.as_ref()) + .await + .map_err(ChartError::BlockscoutDB)? + // no transactions for yesterday + .unwrap_or_else(|| Value { value: 0.0 }); + + Ok(TimespanValue { + timespan: update_time.date_naive(), + value: data.value, + }) + } +} + +pub type AverageTxnFee24hRemote = RemoteDatabaseSource; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "averageTxnFee24h".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillZero + } +} + +pub type AverageTxnFee24h = + DirectPointLocalDbChartSource, Properties>; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{point_construction::dt, simple_test::simple_test_counter}; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_average_txns_fee_1() { + simple_test_counter::( + "update_average_txns_fee_1", + "0.000023592592569", + None, + ) + .await; + } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_average_txns_fee_2() { + simple_test_counter::( + "update_average_txns_fee_2", + "0.00006938997814411765", + Some(dt("2022-11-11T16:00:00")), + ) + .await; + } +} diff --git a/stats/stats/src/charts/counters/mod.rs b/stats/stats/src/charts/counters/mod.rs index f9a59ec36..22357dffc 100644 --- a/stats/stats/src/charts/counters/mod.rs +++ b/stats/stats/src/charts/counters/mod.rs @@ -1,7 +1,10 @@ mod average_block_time; +mod average_txn_fee_24h; mod completed_txns; mod last_new_contracts; mod last_new_verified_contracts; +mod new_txns_24h; +mod pending_txns; mod total_accounts; mod total_addresses; mod total_blocks; @@ -12,6 +15,7 @@ mod total_operational_txns; mod total_tokens; mod total_txns; mod total_verified_contracts; +mod txns_fee_24h; mod yesterday_txns; #[cfg(test)] diff --git a/stats/stats/src/charts/counters/new_txns_24h.rs b/stats/stats/src/charts/counters/new_txns_24h.rs new file mode 100644 index 000000000..85707f0c9 --- /dev/null +++ b/stats/stats/src/charts/counters/new_txns_24h.rs @@ -0,0 +1,147 @@ +use std::ops::Range; + +use crate::{ + data_source::{ + kinds::{ + local_db::DirectPointLocalDbChartSource, + remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, + }, + types::BlockscoutMigrations, + UpdateContext, + }, + range::{inclusive_range_to_exclusive, UniversalRange}, + types::TimespanValue, + utils::sql_with_range_filter_opt, + ChartError, ChartProperties, MissingDatePolicy, Named, +}; +use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; +use entity::sea_orm_active_enums::ChartType; +use sea_orm::{DbBackend, FromQueryResult, Statement}; + +/// `NewTxnsStatement` but without `group by` +pub struct NewTxnsUngroupedStatement; + +impl StatementFromRange for NewTxnsUngroupedStatement { + fn get_statement( + range: Option>>, + completed_migrations: &BlockscoutMigrations, + ) -> Statement { + // do not filter by `!= to_timestamp(0)` because + // 1. it allows to use index `transactions_block_consensus_index` + // 2. there is no reason not to count genesis transactions + if completed_migrations.denormalization { + sql_with_range_filter_opt!( + DbBackend::Postgres, + r#" + SELECT + COUNT(*)::TEXT as value + FROM transactions t + WHERE + t.block_consensus = true {filter}; + "#, + [], + "t.block_timestamp", + range + ) + } else { + sql_with_range_filter_opt!( + DbBackend::Postgres, + r#" + SELECT + COUNT(*)::TEXT as value + FROM transactions t + JOIN blocks b ON t.block_hash = b.hash + WHERE + b.consensus = true {filter}; + "#, + [], + "b.timestamp", + range + ) + } + } +} + +#[derive(FromQueryResult)] +struct Value { + value: String, +} + +pub struct NewTxns24hQuery; + +impl RemoteQueryBehaviour for NewTxns24hQuery { + type Output = TimespanValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result { + let update_time = cx.time; + let range_24h = update_time + .checked_sub_signed(TimeDelta::hours(24)) + .unwrap_or(DateTime::::MIN_UTC)..=update_time; + let query = NewTxnsUngroupedStatement::get_statement( + Some(inclusive_range_to_exclusive(range_24h)), + &cx.blockscout_applied_migrations, + ); + let data = Value::find_by_statement(query) + .one(cx.blockscout.connection.as_ref()) + .await + .map_err(ChartError::BlockscoutDB)? + // no transactions for yesterday + .unwrap_or_else(|| Value { + value: "0".to_string(), + }); + Ok(TimespanValue { + timespan: update_time.date_naive(), + value: data.value, + }) + } +} + +pub type NewTxns24hRemote = RemoteDatabaseSource; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "newTxns24h".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } +} + +pub type NewTxns24h = DirectPointLocalDbChartSource; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{point_construction::dt, simple_test::simple_test_counter}; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_new_txns_24h_1() { + simple_test_counter::("update_new_txns_24h_1", "1", None).await; + } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_new_txns_24h_2() { + simple_test_counter::( + "update_new_txns_24h_2", + // block at `2022-11-11T00:00:00` is not counted because sql is not that precise :/ + "12", + Some(dt("2022-11-11T00:00:00")), + ) + .await; + } +} diff --git a/stats/stats/src/charts/counters/pending_txns.rs b/stats/stats/src/charts/counters/pending_txns.rs new file mode 100644 index 000000000..51bce5ff0 --- /dev/null +++ b/stats/stats/src/charts/counters/pending_txns.rs @@ -0,0 +1,105 @@ +use crate::{ + data_source::{ + kinds::{ + local_db::DirectPointLocalDbChartSource, + remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour}, + }, + UpdateContext, + }, + range::UniversalRange, + types::TimespanValue, + ChartError, ChartProperties, MissingDatePolicy, Named, +}; + +use blockscout_db::entity::transactions; +use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; +use entity::sea_orm_active_enums::ChartType; +use sea_orm::{ + ColumnTrait, DbBackend, EntityTrait, FromQueryResult, QueryFilter, QuerySelect, QueryTrait, + Statement, +}; + +pub struct PendingTxnsStatement; + +impl PendingTxnsStatement { + fn get_statement(inserted_from: DateTime) -> Statement { + transactions::Entity::find() + .select_only() + .filter(transactions::Column::BlockHash.is_null()) + .filter(transactions::Column::InsertedAt.gte(inserted_from)) + .column_as(transactions::Column::Hash.count(), "value") + .build(DbBackend::Postgres) + } +} + +#[derive(FromQueryResult)] +struct Value { + value: i64, +} + +pub struct PendingTxnsQuery; + +impl RemoteQueryBehaviour for PendingTxnsQuery { + type Output = TimespanValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result { + let update_time = cx.time; + let query = PendingTxnsStatement::get_statement( + update_time + .checked_sub_signed(TimeDelta::minutes(30)) + .unwrap_or(DateTime::::MIN_UTC), + ); + let data = Value::find_by_statement(query) + .one(cx.blockscout.connection.as_ref()) + .await + .map_err(ChartError::BlockscoutDB)? + .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; + Ok(TimespanValue { + timespan: update_time.date_naive(), + value: data.value.to_string(), + }) + } +} + +pub type PendingTxnsRemote = RemoteDatabaseSource; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "pendingTxns".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } +} + +pub type PendingTxns = DirectPointLocalDbChartSource; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::simple_test::simple_test_counter_with_migration_variants; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_pending_txns() { + simple_test_counter_with_migration_variants::( + "update_pending_txns", + "0", + None, + ) + .await; + } +} diff --git a/stats/stats/src/charts/counters/txns_fee_24h.rs b/stats/stats/src/charts/counters/txns_fee_24h.rs new file mode 100644 index 000000000..db996c1a6 --- /dev/null +++ b/stats/stats/src/charts/counters/txns_fee_24h.rs @@ -0,0 +1,173 @@ +use std::ops::Range; + +use crate::{ + data_source::{ + kinds::{ + local_db::DirectPointLocalDbChartSource, + remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, + }, + types::BlockscoutMigrations, + UpdateContext, + }, + range::{inclusive_range_to_exclusive, UniversalRange}, + types::TimespanValue, + utils::{produce_filter_and_values, sql_with_range_filter_opt}, + ChartError, ChartProperties, MissingDatePolicy, Named, +}; +use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; +use entity::sea_orm_active_enums::ChartType; +use sea_orm::{DbBackend, FromQueryResult, Statement}; + +const ETHER: i64 = i64::pow(10, 18); + +pub struct TxnsFeeUngroupedStatement; + +impl StatementFromRange for TxnsFeeUngroupedStatement { + fn get_statement( + range: Option>>, + completed_migrations: &BlockscoutMigrations, + ) -> Statement { + if completed_migrations.denormalization { + // TODO: consider supporting such case in macro ? + let mut args = vec![ETHER.into()]; + let (tx_filter, new_args) = + produce_filter_and_values(range.clone(), "t.block_timestamp", args.len() + 1); + args.extend(new_args); + let (block_filter, new_args) = + produce_filter_and_values(range.clone(), "b.timestamp", args.len() + 1); + args.extend(new_args); + let sql = format!( + r#" + SELECT + (SUM( + t_filtered.gas_used * + COALESCE( + t_filtered.gas_price, + b.base_fee_per_gas + LEAST( + t_filtered.max_priority_fee_per_gas, + t_filtered.max_fee_per_gas - b.base_fee_per_gas + ) + ) + ) / $1)::FLOAT as value + FROM ( + SELECT * from transactions t + WHERE + t.block_consensus = true AND + t.block_timestamp != to_timestamp(0) {tx_filter} + ) as t_filtered + JOIN blocks b ON t_filtered.block_hash = b.hash + WHERE + b.timestamp != to_timestamp(0) AND + b.consensus = true {block_filter} + "#, + ); + Statement::from_sql_and_values(DbBackend::Postgres, sql, args) + } else { + sql_with_range_filter_opt!( + DbBackend::Postgres, + r#" + SELECT + (SUM( + t.gas_used * + COALESCE( + t.gas_price, + b.base_fee_per_gas + LEAST( + t.max_priority_fee_per_gas, + t.max_fee_per_gas - b.base_fee_per_gas + ) + ) + ) / $1)::FLOAT as value + FROM transactions t + JOIN blocks b ON t.block_hash = b.hash + WHERE + b.timestamp != to_timestamp(0) AND + b.consensus = true {filter} + "#, + [ETHER.into()], + "b.timestamp", + range + ) + } + } +} + +#[derive(FromQueryResult)] +struct Value { + value: f64, +} + +pub struct TxnsFee24hQuery; + +impl RemoteQueryBehaviour for TxnsFee24hQuery { + type Output = TimespanValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result { + let update_time = cx.time; + let range_24h = update_time + .checked_sub_signed(TimeDelta::hours(24)) + .unwrap_or(DateTime::::MIN_UTC)..=update_time; + let query = TxnsFeeUngroupedStatement::get_statement( + Some(inclusive_range_to_exclusive(range_24h)), + &cx.blockscout_applied_migrations, + ); + let data = Value::find_by_statement(query) + .one(cx.blockscout.connection.as_ref()) + .await + .map_err(ChartError::BlockscoutDB)? + // no transactions for yesterday + .unwrap_or_else(|| Value { value: 0.0 }); + Ok(TimespanValue { + timespan: update_time.date_naive(), + value: data.value.to_string(), + }) + } +} + +pub type TxnsFee24hRemote = RemoteDatabaseSource; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "txnsFee24h".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } +} + +pub type TxnsFee24h = DirectPointLocalDbChartSource; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{point_construction::dt, simple_test::simple_test_counter}; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_txns_fee_24h_1() { + simple_test_counter::("update_txns_fee_24h_1", "0.000023592592569", None).await; + } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_txns_fee_24h_2() { + simple_test_counter::( + "update_txns_fee_24h_2", + "0.000495444443949", + Some(dt("2022-11-11T00:00:00")), + ) + .await; + } +} From 7a453f66f46bab16a3959873ab264f9960dfb813 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Tue, 17 Dec 2024 17:27:29 +0900 Subject: [PATCH 03/17] add counters to configs --- stats/config/charts.json | 5 +++++ stats/config/layout.json | 6 +++++- stats/config/update_groups.json | 4 +++- stats/stats-server/src/runtime_setup.rs | 2 ++ stats/stats-server/tests/it/counters.rs | 4 ++++ stats/stats/src/charts/counters/mod.rs | 4 ++++ stats/stats/src/update_groups.rs | 5 +++++ 7 files changed, 28 insertions(+), 2 deletions(-) diff --git a/stats/config/charts.json b/stats/config/charts.json index 056b24224..ddfb1a6f2 100644 --- a/stats/config/charts.json +++ b/stats/config/charts.json @@ -18,6 +18,11 @@ "description": "Average time taken in seconds for a block to be included in the blockchain", "units": "s" }, + "average_txn_fee_24h": { + "title": "Avg. transaction fee", + "description": "Average amount of {{native_coin_symbol}} spent on gas fees per transaction within last 24 hours", + "units": "{{native_coin_symbol}}" + }, "completed_txns": { "title": "Completed txns", "description": "Number of transactions with success status" diff --git a/stats/config/layout.json b/stats/config/layout.json index 56747311f..62263bd51 100644 --- a/stats/config/layout.json +++ b/stats/config/layout.json @@ -14,7 +14,11 @@ "total_txns", "total_operational_txns", "total_verified_contracts", - "yesterday_txns" + "yesterday_txns", + "new_txns_24h", + "pending_txns", + "txns_fee_24h", + "average_txn_fee_24h" ], "line_chart_categories": [ { diff --git a/stats/config/update_groups.json b/stats/config/update_groups.json index c823856a1..02299df0b 100644 --- a/stats/config/update_groups.json +++ b/stats/config/update_groups.json @@ -3,6 +3,7 @@ "active_accounts_group": "0 0 4 * * * *", "average_block_time_group": "0 */10 * * * * *", "completed_txns_group": "0 5 */3 * * * *", + "pending_txns_group": "0 */5 * * * * *", "total_addresses_group": "0 0,30 * * * * *", "total_blocks_group": "0 0 */2 * * * *", "total_tokens_group": "0 0 18 * * * *", @@ -37,6 +38,7 @@ "new_txns_group": "0 10 */3 * * * *", "new_verified_contracts_group": "0 30 */3 * * * *", "native_coin_holders_growth_group": "0 0 7,17,22 * * * *", - "new_native_coin_transfers_group": "0 0 3,13 * * * *" + "new_native_coin_transfers_group": "0 0 3,13 * * * *", + "txns_stats_24h_group": "0 30 * * * * *" } } \ No newline at end of file diff --git a/stats/stats-server/src/runtime_setup.rs b/stats/stats-server/src/runtime_setup.rs index d15c670dc..c28bfb275 100644 --- a/stats/stats-server/src/runtime_setup.rs +++ b/stats/stats-server/src/runtime_setup.rs @@ -257,6 +257,7 @@ impl RuntimeSetup { Arc::new(ActiveAccountsGroup), Arc::new(AverageBlockTimeGroup), Arc::new(CompletedTxnsGroup), + Arc::new(PendingTxnsGroup), Arc::new(TotalAddressesGroup), Arc::new(TotalBlocksGroup), Arc::new(TotalTokensGroup), @@ -294,6 +295,7 @@ impl RuntimeSetup { Arc::new(NewVerifiedContractsGroup), Arc::new(NativeCoinHoldersGrowthGroup), Arc::new(NewNativeCoinTransfersGroup), + Arc::new(TxnsStats24hGroup), ] } diff --git a/stats/stats-server/tests/it/counters.rs b/stats/stats-server/tests/it/counters.rs index ddb05a696..fad97e267 100644 --- a/stats/stats-server/tests/it/counters.rs +++ b/stats/stats-server/tests/it/counters.rs @@ -59,6 +59,10 @@ async fn test_counters_ok() { "totalTxns", "totalVerifiedContracts", "yesterdayTxns", + "new_txns_24h", + "pending_txns", + "txns_fee_24h", + "average_txn_fee_24h", ] .into_iter() .collect(); diff --git a/stats/stats/src/charts/counters/mod.rs b/stats/stats/src/charts/counters/mod.rs index 22357dffc..37521eb34 100644 --- a/stats/stats/src/charts/counters/mod.rs +++ b/stats/stats/src/charts/counters/mod.rs @@ -22,9 +22,12 @@ mod yesterday_txns; mod mock; pub use average_block_time::AverageBlockTime; +pub use average_txn_fee_24h::AverageTxnFee24h; pub use completed_txns::CompletedTxns; pub use last_new_contracts::LastNewContracts; pub use last_new_verified_contracts::LastNewVerifiedContracts; +pub use new_txns_24h::NewTxns24h; +pub use pending_txns::PendingTxns; pub use total_accounts::TotalAccounts; pub use total_addresses::TotalAddresses; pub use total_blocks::{TotalBlocks, TotalBlocksInt}; @@ -35,6 +38,7 @@ pub use total_operational_txns::TotalOperationalTxns; pub use total_tokens::TotalTokens; pub use total_txns::{TotalTxns, TotalTxnsInt}; pub use total_verified_contracts::TotalVerifiedContracts; +pub use txns_fee_24h::TxnsFee24h; pub use yesterday_txns::YesterdayTxns; #[cfg(test)] diff --git a/stats/stats/src/update_groups.rs b/stats/stats/src/update_groups.rs index 9bdbb805b..a467f0268 100644 --- a/stats/stats/src/update_groups.rs +++ b/stats/stats/src/update_groups.rs @@ -20,6 +20,7 @@ singleton_groups!( ActiveAccounts, AverageBlockTime, CompletedTxns, + PendingTxns, TotalAddresses, TotalBlocks, TotalTxns, @@ -141,6 +142,10 @@ construct_update_group!(TxnsSuccessRateGroup { ] }); +construct_update_group!(TxnsStats24hGroup { + charts: [AverageTxnFee24h, NewTxns24h, TxnsFee24h,] +}); + construct_update_group!(NewAccountsGroup { charts: [ NewAccounts, From 2c36b5ef3956ac6fc4cc151319b34d95ab4e8164 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Tue, 17 Dec 2024 21:33:56 +0900 Subject: [PATCH 04/17] fix --- stats/config/charts.json | 13 +++++++++++ stats/stats-server/tests/it/counters.rs | 8 +++---- .../charts/counters/average_txn_fee_24h.rs | 23 +++++++++++++++---- .../stats/src/charts/counters/new_txns_24h.rs | 11 +++++++++ .../stats/src/charts/counters/txns_fee_24h.rs | 23 +++++++++++++++---- 5 files changed, 66 insertions(+), 12 deletions(-) diff --git a/stats/config/charts.json b/stats/config/charts.json index ddfb1a6f2..0a95c3dce 100644 --- a/stats/config/charts.json +++ b/stats/config/charts.json @@ -18,6 +18,19 @@ "description": "Average time taken in seconds for a block to be included in the blockchain", "units": "s" }, + "new_txns_24h": { + "title": "Transactions", + "description": "Number of new transactions within last 24 hours" + }, + "pending_txns": { + "title": "Pending transactions", + "description": "" + }, + "txns_fee_24h": { + "title": "Transactions fees", + "description": "Sum of {{native_coin_symbol}} spent on gas fees within last 24 hours", + "units": "{{native_coin_symbol}}" + }, "average_txn_fee_24h": { "title": "Avg. transaction fee", "description": "Average amount of {{native_coin_symbol}} spent on gas fees per transaction within last 24 hours", diff --git a/stats/stats-server/tests/it/counters.rs b/stats/stats-server/tests/it/counters.rs index fad97e267..e1100e708 100644 --- a/stats/stats-server/tests/it/counters.rs +++ b/stats/stats-server/tests/it/counters.rs @@ -59,10 +59,10 @@ async fn test_counters_ok() { "totalTxns", "totalVerifiedContracts", "yesterdayTxns", - "new_txns_24h", - "pending_txns", - "txns_fee_24h", - "average_txn_fee_24h", + "newTxns24h", + "pendingTxns", + "txnsFee24h", + "averageTxnFee24h", ] .into_iter() .collect(); diff --git a/stats/stats/src/charts/counters/average_txn_fee_24h.rs b/stats/stats/src/charts/counters/average_txn_fee_24h.rs index eef6c97d5..331b57b9d 100644 --- a/stats/stats/src/charts/counters/average_txn_fee_24h.rs +++ b/stats/stats/src/charts/counters/average_txn_fee_24h.rs @@ -95,7 +95,9 @@ impl StatementFromRange for AverageTxnFeeUngroupedStatement { #[derive(FromQueryResult)] struct Value { - value: f64, + // if there are no transactions/blocks - the row is still returned + // but with null value + value: Option, } pub struct AverageTxnFee24hQuery; @@ -116,16 +118,18 @@ impl RemoteQueryBehaviour for AverageTxnFee24hQuery { &cx.blockscout_applied_migrations, ); - let data = Value::find_by_statement(query) + let value = Value::find_by_statement(query) .one(cx.blockscout.connection.as_ref()) .await .map_err(ChartError::BlockscoutDB)? + .map(|v| v.value) + .flatten() // no transactions for yesterday - .unwrap_or_else(|| Value { value: 0.0 }); + .unwrap_or_else(|| 0.0); Ok(TimespanValue { timespan: update_time.date_naive(), - value: data.value, + value: value, }) } } @@ -180,4 +184,15 @@ mod tests { ) .await; } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_average_txns_fee_3() { + simple_test_counter::( + "update_average_txns_fee_3", + "0", + Some(dt("2024-10-10T00:00:00")), + ) + .await; + } } diff --git a/stats/stats/src/charts/counters/new_txns_24h.rs b/stats/stats/src/charts/counters/new_txns_24h.rs index 85707f0c9..82bd38a84 100644 --- a/stats/stats/src/charts/counters/new_txns_24h.rs +++ b/stats/stats/src/charts/counters/new_txns_24h.rs @@ -144,4 +144,15 @@ mod tests { ) .await; } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_new_txns_24h_3() { + simple_test_counter::( + "update_new_txns_24h_3", + "0", + Some(dt("2024-11-11T00:00:00")), + ) + .await; + } } diff --git a/stats/stats/src/charts/counters/txns_fee_24h.rs b/stats/stats/src/charts/counters/txns_fee_24h.rs index db996c1a6..23d95b782 100644 --- a/stats/stats/src/charts/counters/txns_fee_24h.rs +++ b/stats/stats/src/charts/counters/txns_fee_24h.rs @@ -93,7 +93,9 @@ impl StatementFromRange for TxnsFeeUngroupedStatement { #[derive(FromQueryResult)] struct Value { - value: f64, + // if there are no transactions/blocks - the row is still returned + // but with null value + value: Option, } pub struct TxnsFee24hQuery; @@ -113,15 +115,17 @@ impl RemoteQueryBehaviour for TxnsFee24hQuery { Some(inclusive_range_to_exclusive(range_24h)), &cx.blockscout_applied_migrations, ); - let data = Value::find_by_statement(query) + let value = Value::find_by_statement(query) .one(cx.blockscout.connection.as_ref()) .await .map_err(ChartError::BlockscoutDB)? + .map(|v| v.value) + .flatten() // no transactions for yesterday - .unwrap_or_else(|| Value { value: 0.0 }); + .unwrap_or_else(|| 0.0); Ok(TimespanValue { timespan: update_time.date_naive(), - value: data.value.to_string(), + value: value.to_string(), }) } } @@ -170,4 +174,15 @@ mod tests { ) .await; } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_txns_fee_24h_3() { + simple_test_counter::( + "update_txns_fee_24h_3", + "0", + Some(dt("2024-11-11T00:00:00")), + ) + .await; + } } From 8c37d71ead46551b112dc986eb0b111ff2622186 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Tue, 17 Dec 2024 21:43:31 +0900 Subject: [PATCH 05/17] bump blockscout-service-launcher --- stats/Cargo.lock | 4 ++-- stats/Cargo.toml | 2 +- stats/config/charts.json | 2 +- stats/stats/Cargo.toml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stats/Cargo.lock b/stats/Cargo.lock index 0b63993cf..58b8477cb 100644 --- a/stats/Cargo.lock +++ b/stats/Cargo.lock @@ -852,9 +852,9 @@ source = "git+https://github.com/blockscout/blockscout-rs?rev=980634e#980634e7ad [[package]] name = "blockscout-service-launcher" -version = "0.13.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27eb8c1c7261a8097dfa9746c7f5ba8bebef2481791a33078b28775c56940b74" +checksum = "98faf66096af3a77362da42c2861e59427187b8f1687f482cb4d04465176d3d0" dependencies = [ "actix-cors", "actix-web", diff --git a/stats/Cargo.toml b/stats/Cargo.toml index 82cf74972..cd76553fa 100644 --- a/stats/Cargo.toml +++ b/stats/Cargo.toml @@ -11,7 +11,7 @@ members = [ [workspace.dependencies] blockscout-client = { git = "https://github.com/blockscout/blockscout-rs/", rev = "506b821" } -blockscout-service-launcher = { version = "0.13.1" } +blockscout-service-launcher = { version = "0.15.0" } rstest = "0.23.0" trait-variant = "0.1.2" wiremock = "0.6.2" diff --git a/stats/config/charts.json b/stats/config/charts.json index 0a95c3dce..a2b3c5308 100644 --- a/stats/config/charts.json +++ b/stats/config/charts.json @@ -24,7 +24,7 @@ }, "pending_txns": { "title": "Pending transactions", - "description": "" + "description": " " }, "txns_fee_24h": { "title": "Transactions fees", diff --git a/stats/stats/Cargo.toml b/stats/stats/Cargo.toml index ead654035..52d34f84c 100644 --- a/stats/stats/Cargo.toml +++ b/stats/stats/Cargo.toml @@ -34,7 +34,7 @@ rust_decimal = "1.27" # Dependencies for test-utils only pretty_assertions = { version= "1.2", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } -blockscout-service-launcher = { version = "0.13.1", features = [ "database-0_12", "test-database" ], optional = true } +blockscout-service-launcher = { workspace = true, features = [ "database-0_12", "test-database" ], optional = true } wiremock = { workspace = true, optional = true } [dev-dependencies] @@ -49,7 +49,7 @@ rust_decimal_macros = "1.27" # test-utils pretty_assertions = "1.2" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -blockscout-service-launcher = { version = "0.13.1", features = [ "database-0_12", "test-database" ] } +blockscout-service-launcher = { workspace = true, features = [ "database-0_12", "test-database" ] } wiremock = { workspace = true } [features] From b4631614a40e94830b3effc120f8af745c75cce6 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 18 Dec 2024 19:33:33 +0900 Subject: [PATCH 06/17] extract pull 24 hour impl into one common --- .../charts/counters/average_txn_fee_24h.rs | 55 ++----------------- .../stats/src/charts/counters/new_txns_24h.rs | 53 +++--------------- .../stats/src/charts/counters/txns_fee_24h.rs | 55 +++---------------- .../src/data_source/kinds/remote_db/mod.rs | 2 +- .../data_source/kinds/remote_db/query/mod.rs | 2 +- .../data_source/kinds/remote_db/query/one.rs | 55 ++++++++++++++++++- 6 files changed, 74 insertions(+), 148 deletions(-) diff --git a/stats/stats/src/charts/counters/average_txn_fee_24h.rs b/stats/stats/src/charts/counters/average_txn_fee_24h.rs index 331b57b9d..654ce064c 100644 --- a/stats/stats/src/charts/counters/average_txn_fee_24h.rs +++ b/stats/stats/src/charts/counters/average_txn_fee_24h.rs @@ -5,19 +5,16 @@ use crate::{ kinds::{ data_manipulation::map::MapToString, local_db::DirectPointLocalDbChartSource, - remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, + remote_db::{PullOne24h, RemoteDatabaseSource, StatementFromRange}, }, types::BlockscoutMigrations, - UpdateContext, }, - range::{inclusive_range_to_exclusive, UniversalRange}, - types::TimespanValue, utils::{produce_filter_and_values, sql_with_range_filter_opt}, - ChartError, ChartProperties, MissingDatePolicy, Named, + ChartProperties, MissingDatePolicy, Named, }; -use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{DbBackend, FromQueryResult, Statement}; +use sea_orm::{DbBackend, Statement}; const ETHER: i64 = i64::pow(10, 18); @@ -93,48 +90,8 @@ impl StatementFromRange for AverageTxnFeeUngroupedStatement { } } -#[derive(FromQueryResult)] -struct Value { - // if there are no transactions/blocks - the row is still returned - // but with null value - value: Option, -} - -pub struct AverageTxnFee24hQuery; - -impl RemoteQueryBehaviour for AverageTxnFee24hQuery { - type Output = TimespanValue; - - async fn query_data( - cx: &UpdateContext<'_>, - _range: UniversalRange>, - ) -> Result { - let update_time = cx.time; - let range_24h = update_time - .checked_sub_signed(TimeDelta::hours(24)) - .unwrap_or(DateTime::::MIN_UTC)..=update_time; - let query = AverageTxnFeeUngroupedStatement::get_statement( - Some(inclusive_range_to_exclusive(range_24h)), - &cx.blockscout_applied_migrations, - ); - - let value = Value::find_by_statement(query) - .one(cx.blockscout.connection.as_ref()) - .await - .map_err(ChartError::BlockscoutDB)? - .map(|v| v.value) - .flatten() - // no transactions for yesterday - .unwrap_or_else(|| 0.0); - - Ok(TimespanValue { - timespan: update_time.date_naive(), - value: value, - }) - } -} - -pub type AverageTxnFee24hRemote = RemoteDatabaseSource; +pub type AverageTxnFee24hRemote = + RemoteDatabaseSource>; pub struct Properties; diff --git a/stats/stats/src/charts/counters/new_txns_24h.rs b/stats/stats/src/charts/counters/new_txns_24h.rs index 82bd38a84..a6aecbfcb 100644 --- a/stats/stats/src/charts/counters/new_txns_24h.rs +++ b/stats/stats/src/charts/counters/new_txns_24h.rs @@ -4,19 +4,17 @@ use crate::{ data_source::{ kinds::{ local_db::DirectPointLocalDbChartSource, - remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, + remote_db::{ + PullOne24h, RemoteDatabaseSource, StatementFromRange, + }, }, types::BlockscoutMigrations, - UpdateContext, }, - range::{inclusive_range_to_exclusive, UniversalRange}, - types::TimespanValue, - utils::sql_with_range_filter_opt, - ChartError, ChartProperties, MissingDatePolicy, Named, + utils::sql_with_range_filter_opt, ChartProperties, MissingDatePolicy, Named, }; -use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{DbBackend, FromQueryResult, Statement}; +use sea_orm::{DbBackend, Statement}; /// `NewTxnsStatement` but without `group by` pub struct NewTxnsUngroupedStatement; @@ -62,44 +60,7 @@ impl StatementFromRange for NewTxnsUngroupedStatement { } } -#[derive(FromQueryResult)] -struct Value { - value: String, -} - -pub struct NewTxns24hQuery; - -impl RemoteQueryBehaviour for NewTxns24hQuery { - type Output = TimespanValue; - - async fn query_data( - cx: &UpdateContext<'_>, - _range: UniversalRange>, - ) -> Result { - let update_time = cx.time; - let range_24h = update_time - .checked_sub_signed(TimeDelta::hours(24)) - .unwrap_or(DateTime::::MIN_UTC)..=update_time; - let query = NewTxnsUngroupedStatement::get_statement( - Some(inclusive_range_to_exclusive(range_24h)), - &cx.blockscout_applied_migrations, - ); - let data = Value::find_by_statement(query) - .one(cx.blockscout.connection.as_ref()) - .await - .map_err(ChartError::BlockscoutDB)? - // no transactions for yesterday - .unwrap_or_else(|| Value { - value: "0".to_string(), - }); - Ok(TimespanValue { - timespan: update_time.date_naive(), - value: data.value, - }) - } -} - -pub type NewTxns24hRemote = RemoteDatabaseSource; +pub type NewTxns24hRemote = RemoteDatabaseSource>; pub struct Properties; diff --git a/stats/stats/src/charts/counters/txns_fee_24h.rs b/stats/stats/src/charts/counters/txns_fee_24h.rs index 23d95b782..945a58378 100644 --- a/stats/stats/src/charts/counters/txns_fee_24h.rs +++ b/stats/stats/src/charts/counters/txns_fee_24h.rs @@ -3,20 +3,18 @@ use std::ops::Range; use crate::{ data_source::{ kinds::{ + data_manipulation::map::MapToString, local_db::DirectPointLocalDbChartSource, - remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, + remote_db::{PullOne24h, RemoteDatabaseSource, StatementFromRange}, }, types::BlockscoutMigrations, - UpdateContext, }, - range::{inclusive_range_to_exclusive, UniversalRange}, - types::TimespanValue, utils::{produce_filter_and_values, sql_with_range_filter_opt}, - ChartError, ChartProperties, MissingDatePolicy, Named, + ChartProperties, MissingDatePolicy, Named, }; -use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{DbBackend, FromQueryResult, Statement}; +use sea_orm::{DbBackend, Statement}; const ETHER: i64 = i64::pow(10, 18); @@ -91,46 +89,7 @@ impl StatementFromRange for TxnsFeeUngroupedStatement { } } -#[derive(FromQueryResult)] -struct Value { - // if there are no transactions/blocks - the row is still returned - // but with null value - value: Option, -} - -pub struct TxnsFee24hQuery; - -impl RemoteQueryBehaviour for TxnsFee24hQuery { - type Output = TimespanValue; - - async fn query_data( - cx: &UpdateContext<'_>, - _range: UniversalRange>, - ) -> Result { - let update_time = cx.time; - let range_24h = update_time - .checked_sub_signed(TimeDelta::hours(24)) - .unwrap_or(DateTime::::MIN_UTC)..=update_time; - let query = TxnsFeeUngroupedStatement::get_statement( - Some(inclusive_range_to_exclusive(range_24h)), - &cx.blockscout_applied_migrations, - ); - let value = Value::find_by_statement(query) - .one(cx.blockscout.connection.as_ref()) - .await - .map_err(ChartError::BlockscoutDB)? - .map(|v| v.value) - .flatten() - // no transactions for yesterday - .unwrap_or_else(|| 0.0); - Ok(TimespanValue { - timespan: update_time.date_naive(), - value: value.to_string(), - }) - } -} - -pub type TxnsFee24hRemote = RemoteDatabaseSource; +pub type TxnsFee24hRemote = RemoteDatabaseSource>; pub struct Properties; @@ -151,7 +110,7 @@ impl ChartProperties for Properties { } } -pub type TxnsFee24h = DirectPointLocalDbChartSource; +pub type TxnsFee24h = DirectPointLocalDbChartSource, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/data_source/kinds/remote_db/mod.rs b/stats/stats/src/data_source/kinds/remote_db/mod.rs index 6d54909e3..e326c5ce9 100644 --- a/stats/stats/src/data_source/kinds/remote_db/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/mod.rs @@ -34,7 +34,7 @@ use crate::{ }; pub use query::{ - PullAllWithAndSort, PullEachWith, PullOne, StatementForOne, StatementFromRange, + PullAllWithAndSort, PullEachWith, PullOne, PullOne24h, StatementForOne, StatementFromRange, StatementFromTimespan, }; diff --git a/stats/stats/src/data_source/kinds/remote_db/query/mod.rs b/stats/stats/src/data_source/kinds/remote_db/query/mod.rs index 35d2189b8..9a3b114a7 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/mod.rs @@ -4,4 +4,4 @@ mod one; pub use all::{PullAllWithAndSort, StatementFromRange}; pub use each::{PullEachWith, StatementFromTimespan}; -pub use one::{PullOne, StatementForOne}; +pub use one::{PullOne, PullOne24h, StatementForOne}; diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index 1a1dcedc2..c80a4113c 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -1,18 +1,20 @@ use std::marker::{PhantomData, Send}; -use chrono::{DateTime, Utc}; -use sea_orm::{FromQueryResult, Statement}; +use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; +use sea_orm::{FromQueryResult, Statement, TryGetable}; use crate::{ data_source::{ kinds::remote_db::RemoteQueryBehaviour, types::{BlockscoutMigrations, UpdateContext}, }, - range::UniversalRange, + range::{inclusive_range_to_exclusive, UniversalRange}, types::TimespanValue, ChartError, }; +use super::StatementFromRange; + pub trait StatementForOne { fn get_statement(completed_migrations: &BlockscoutMigrations) -> Statement; } @@ -52,3 +54,50 @@ where Ok(data) } } + +#[derive(FromQueryResult)] +struct FromQueryValue { + value: V, +} + +/// Retrieve a single value `Value` for the period of last 24h +/// according to provided statemnt `S`. +/// +/// The point will be for the time of update +pub struct PullOne24h(PhantomData<(S, Value)>) +where + S: StatementFromRange, + Value: Send + TryGetable; + +impl RemoteQueryBehaviour for PullOne24h +where + S: StatementFromRange, + Value: Send + TryGetable, +{ + type Output = TimespanValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result, ChartError> { + let update_time = cx.time; + let range_24h = update_time + .checked_sub_signed(TimeDelta::hours(24)) + .unwrap_or(DateTime::::MIN_UTC)..=update_time; + let query = S::get_statement( + Some(inclusive_range_to_exclusive(range_24h)), + &cx.blockscout_applied_migrations, + ); + + let value = FromQueryValue::::find_by_statement(query) + .one(cx.blockscout.connection.as_ref()) + .await + .map_err(ChartError::BlockscoutDB)? + .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; + + Ok(TimespanValue { + timespan: update_time.date_naive(), + value: value.value, + }) + } +} From bbaf8932c69ecfce550bc17b54e7589f5ed4c54b Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Wed, 18 Dec 2024 20:08:15 +0900 Subject: [PATCH 07/17] fix no blocks behaviour --- .../charts/counters/average_txn_fee_24h.rs | 9 ++-- .../stats/src/charts/counters/txns_fee_24h.rs | 11 ++-- .../kinds/data_manipulation/map/mod.rs | 2 + .../kinds/data_manipulation/map/unwrap_or.rs | 51 +++++++++++++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 stats/stats/src/data_source/kinds/data_manipulation/map/unwrap_or.rs diff --git a/stats/stats/src/charts/counters/average_txn_fee_24h.rs b/stats/stats/src/charts/counters/average_txn_fee_24h.rs index 654ce064c..171bd08e1 100644 --- a/stats/stats/src/charts/counters/average_txn_fee_24h.rs +++ b/stats/stats/src/charts/counters/average_txn_fee_24h.rs @@ -3,12 +3,13 @@ use std::ops::Range; use crate::{ data_source::{ kinds::{ - data_manipulation::map::MapToString, + data_manipulation::map::{MapToString, UnwrapOr}, local_db::DirectPointLocalDbChartSource, remote_db::{PullOne24h, RemoteDatabaseSource, StatementFromRange}, }, types::BlockscoutMigrations, }, + gettable_const, utils::{produce_filter_and_values, sql_with_range_filter_opt}, ChartProperties, MissingDatePolicy, Named, }; @@ -91,7 +92,7 @@ impl StatementFromRange for AverageTxnFeeUngroupedStatement { } pub type AverageTxnFee24hRemote = - RemoteDatabaseSource>; + RemoteDatabaseSource>>; pub struct Properties; @@ -112,8 +113,10 @@ impl ChartProperties for Properties { } } +gettable_const!(Zero: f64 = 0.0); + pub type AverageTxnFee24h = - DirectPointLocalDbChartSource, Properties>; + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/txns_fee_24h.rs b/stats/stats/src/charts/counters/txns_fee_24h.rs index 945a58378..f74194129 100644 --- a/stats/stats/src/charts/counters/txns_fee_24h.rs +++ b/stats/stats/src/charts/counters/txns_fee_24h.rs @@ -3,12 +3,13 @@ use std::ops::Range; use crate::{ data_source::{ kinds::{ - data_manipulation::map::MapToString, + data_manipulation::map::{MapToString, UnwrapOr}, local_db::DirectPointLocalDbChartSource, remote_db::{PullOne24h, RemoteDatabaseSource, StatementFromRange}, }, types::BlockscoutMigrations, }, + gettable_const, utils::{produce_filter_and_values, sql_with_range_filter_opt}, ChartProperties, MissingDatePolicy, Named, }; @@ -89,7 +90,8 @@ impl StatementFromRange for TxnsFeeUngroupedStatement { } } -pub type TxnsFee24hRemote = RemoteDatabaseSource>; +pub type TxnsFee24hRemote = + RemoteDatabaseSource>>; pub struct Properties; @@ -110,7 +112,10 @@ impl ChartProperties for Properties { } } -pub type TxnsFee24h = DirectPointLocalDbChartSource, Properties>; +gettable_const!(Zero: f64 = 0.0); + +pub type TxnsFee24h = + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs index a0d114b64..feb178a3d 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs @@ -16,10 +16,12 @@ use crate::{ mod parse; mod strip_extension; mod to_string; +mod unwrap_or; pub use parse::MapParseTo; pub use strip_extension::StripExt; pub use to_string::MapToString; +pub use unwrap_or::UnwrapOr; /// Apply `F` to each value queried from data source `D` pub struct Map(PhantomData<(D, F)>) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/unwrap_or.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/unwrap_or.rs new file mode 100644 index 000000000..cb2155256 --- /dev/null +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/unwrap_or.rs @@ -0,0 +1,51 @@ +use std::marker::PhantomData; + +use crate::{data_source::types::Get, types::TimespanValue, ChartError}; + +use super::{Map, MapFunction}; + +pub struct UnwrapOrFunction(PhantomData); + +impl MapFunction>>> + for UnwrapOrFunction +where + Resolution: Send, + Value: Send, + DefaultValue: Get, +{ + type Output = Vec>; + fn function( + inner_data: Vec>>, + ) -> Result { + Ok(inner_data + .into_iter() + .map(|p| TimespanValue { + timespan: p.timespan, + value: p.value.unwrap_or_else(DefaultValue::get), + }) + .collect()) + } +} + +impl MapFunction>> + for UnwrapOrFunction +where + Resolution: Send, + Value: Send, + DefaultValue: Get, +{ + type Output = TimespanValue; + fn function( + inner_data: TimespanValue>, + ) -> Result { + Ok(TimespanValue { + timespan: inner_data.timespan, + value: inner_data.value.unwrap_or_else(DefaultValue::get), + }) + } +} + +/// Returns the data from `D` or provided default value. +/// +/// [`crate::gettable_const`] is useful for defining the default value +pub type UnwrapOr = Map>; From 0ffe3e96a5a9ad1cb9c64019a72f629649e6b39e Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 19 Dec 2024 00:47:41 +0900 Subject: [PATCH 08/17] cache --- stats/stats/src/data_source/types.rs | 122 +++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 58dc2bdd3..0aa612455 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use blockscout_db::entity::migrations_status; use chrono::Utc; use sea_orm::{DatabaseConnection, DbErr, EntityTrait, FromQueryResult, QueryOrder, Statement}; @@ -129,6 +131,90 @@ impl BlockscoutMigrations { } } +pub enum CacheValue { + ValueString(String), + ValueOptionF64(Option), +} + +pub trait Cacheable { + fn from_entry(entry: CacheValue) -> Option + where + Self: Sized; + fn from_entry_ref(entry: &CacheValue) -> Option<&Self> + where + Self: Sized; + fn into_entry(self) -> CacheValue; +} + +macro_rules! impl_cacheable { + ($type: ty, $cache_value_variant:ident) => { + impl Cacheable for $type { + fn from_entry(entry: CacheValue) -> Option + where + Self: Sized, + { + match entry { + CacheValue::$cache_value_variant(s) => Some(s), + _ => None, + } + } + + fn from_entry_ref(entry: &CacheValue) -> Option<&Self> + where + Self: Sized, + { + match entry { + CacheValue::$cache_value_variant(s) => Some(s), + _ => None, + } + } + + fn into_entry(self) -> CacheValue { + CacheValue::$cache_value_variant(self) + } + } + }; +} + +impl_cacheable!(String, ValueString); +impl_cacheable!(Option, ValueOptionF64); + +pub struct UpdateCache { + inner: HashMap, +} + +impl UpdateCache { + pub fn new() -> Self { + Self { + inner: HashMap::new(), + } + } +} + +pub trait StorageFor { + fn insert(&mut self, query: &Statement, value: V) -> Option; + + fn get(&mut self, query: &Statement) -> Option<&V>; +} + +impl UpdateCache { + /// If the cache did not have value for this query present, None is returned. + /// + /// If the cache did have this query present, the value is updated, and the old value is returned. + fn insert(&mut self, query: &Statement, value: V) -> Option { + self.inner + .insert(query.to_string(), value.into_entry()) + .and_then(|e| V::from_entry(e)) + } + + /// Returns reference to a value for this query, if present + fn get(&mut self, query: &Statement) -> Option<&V> { + self.inner + .get(&query.to_string()) + .and_then(|e| V::from_entry_ref(e)) + } +} + pub trait Get { type Value; fn get() -> Self::Value; @@ -156,3 +242,39 @@ macro_rules! gettable_const { } }; } + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + use sea_orm::DbBackend; + + use super::*; + + #[test] + fn cache_works() { + let mut cache = UpdateCache::new(); + let stmt_a = Statement::from_string(DbBackend::Sqlite, "abcde"); + let stmt_b = Statement::from_string(DbBackend::Sqlite, "edcba"); + + let val_1 = Some(1.2); + let val_2 = "kekekek".to_string(); + + cache.insert::>(&stmt_a, val_1); + assert_eq!(cache.get::>(&stmt_a), Some(&val_1)); + assert_eq!(cache.get::(&stmt_a), None); + + cache.insert::>(&stmt_a, None); + assert_eq!(cache.get::>(&stmt_a), Some(&None)); + assert_eq!(cache.get::(&stmt_a), None); + + cache.insert::(&stmt_a, val_2.clone()); + assert_eq!(cache.get::>(&stmt_a), None); + assert_eq!(cache.get::(&stmt_a), Some(&val_2)); + + cache.insert::>(&stmt_b, val_1); + assert_eq!(cache.get::>(&stmt_b), Some(&val_1)); + assert_eq!(cache.get::(&stmt_b), None); + assert_eq!(cache.get::>(&stmt_a), None); + assert_eq!(cache.get::(&stmt_a), Some(&val_2)); + } +} From e1998cd581d80b73c766a0d362850994d953d52d Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 19 Dec 2024 16:33:09 +0900 Subject: [PATCH 09/17] sync update cache --- stats/stats/src/data_source/types.rs | 61 +++++++++++++++------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 0aa612455..8f226970e 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -1,8 +1,9 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use blockscout_db::entity::migrations_status; use chrono::Utc; use sea_orm::{DatabaseConnection, DbErr, EntityTrait, FromQueryResult, QueryOrder, Statement}; +use tokio::sync::Mutex; use tracing::warn; use crate::utils::MarkedDbConnection; @@ -24,6 +25,7 @@ pub struct UpdateContext<'a> { pub db: &'a MarkedDbConnection, pub blockscout: &'a MarkedDbConnection, pub blockscout_applied_migrations: BlockscoutMigrations, + pub cache: UpdateCache, /// Update time pub time: chrono::DateTime, pub force_full: bool, @@ -35,6 +37,7 @@ impl<'a> UpdateContext<'a> { db: value.db, blockscout: value.blockscout, blockscout_applied_migrations: value.blockscout_applied_migrations, + cache: UpdateCache::new(), time: value.update_time_override.unwrap_or_else(Utc::now), force_full: value.force_full, } @@ -131,6 +134,7 @@ impl BlockscoutMigrations { } } +#[derive(Clone, Debug)] pub enum CacheValue { ValueString(String), ValueOptionF64(Option), @@ -179,39 +183,38 @@ macro_rules! impl_cacheable { impl_cacheable!(String, ValueString); impl_cacheable!(Option, ValueOptionF64); +#[derive(Clone, Debug)] pub struct UpdateCache { - inner: HashMap, + inner: Arc>>, } impl UpdateCache { pub fn new() -> Self { Self { - inner: HashMap::new(), + inner: Arc::new(Mutex::new(HashMap::new())), } } } -pub trait StorageFor { - fn insert(&mut self, query: &Statement, value: V) -> Option; - - fn get(&mut self, query: &Statement) -> Option<&V>; -} - impl UpdateCache { /// If the cache did not have value for this query present, None is returned. /// /// If the cache did have this query present, the value is updated, and the old value is returned. - fn insert(&mut self, query: &Statement, value: V) -> Option { + async fn insert(&mut self, query: &Statement, value: V) -> Option { self.inner + .lock() + .await .insert(query.to_string(), value.into_entry()) .and_then(|e| V::from_entry(e)) } - /// Returns reference to a value for this query, if present - fn get(&mut self, query: &Statement) -> Option<&V> { + /// Returns a value for this query, if present + async fn get(&mut self, query: &Statement) -> Option { self.inner + .lock() + .await .get(&query.to_string()) - .and_then(|e| V::from_entry_ref(e)) + .and_then(|e| V::from_entry(e.clone())) } } @@ -250,8 +253,8 @@ mod tests { use super::*; - #[test] - fn cache_works() { + #[tokio::test] + async fn cache_works() { let mut cache = UpdateCache::new(); let stmt_a = Statement::from_string(DbBackend::Sqlite, "abcde"); let stmt_b = Statement::from_string(DbBackend::Sqlite, "edcba"); @@ -259,22 +262,22 @@ mod tests { let val_1 = Some(1.2); let val_2 = "kekekek".to_string(); - cache.insert::>(&stmt_a, val_1); - assert_eq!(cache.get::>(&stmt_a), Some(&val_1)); - assert_eq!(cache.get::(&stmt_a), None); + cache.insert::>(&stmt_a, val_1).await; + assert_eq!(cache.get::>(&stmt_a).await, Some(val_1)); + assert_eq!(cache.get::(&stmt_a).await, None); - cache.insert::>(&stmt_a, None); - assert_eq!(cache.get::>(&stmt_a), Some(&None)); - assert_eq!(cache.get::(&stmt_a), None); + cache.insert::>(&stmt_a, None).await; + assert_eq!(cache.get::>(&stmt_a).await, Some(None)); + assert_eq!(cache.get::(&stmt_a).await, None); - cache.insert::(&stmt_a, val_2.clone()); - assert_eq!(cache.get::>(&stmt_a), None); - assert_eq!(cache.get::(&stmt_a), Some(&val_2)); + cache.insert::(&stmt_a, val_2.clone()).await; + assert_eq!(cache.get::>(&stmt_a).await, None); + assert_eq!(cache.get::(&stmt_a).await, Some(val_2.clone())); - cache.insert::>(&stmt_b, val_1); - assert_eq!(cache.get::>(&stmt_b), Some(&val_1)); - assert_eq!(cache.get::(&stmt_b), None); - assert_eq!(cache.get::>(&stmt_a), None); - assert_eq!(cache.get::(&stmt_a), Some(&val_2)); + cache.insert::>(&stmt_b, val_1).await; + assert_eq!(cache.get::>(&stmt_b).await, Some(val_1)); + assert_eq!(cache.get::(&stmt_b).await, None); + assert_eq!(cache.get::>(&stmt_a).await, None); + assert_eq!(cache.get::(&stmt_a).await, Some(val_2)); } } From 69c0ec5ec15b30b67400ee491013230894457887 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Thu, 19 Dec 2024 19:02:57 +0900 Subject: [PATCH 10/17] cached remote --- stats/stats/Cargo.toml | 1 + stats/stats/src/charts/lines/new_blocks.rs | 20 +-- stats/stats/src/charts/query_dispatch.rs | 6 +- .../data_manipulation/filter_deducible.rs | 8 +- .../data_manipulation/resolutions/average.rs | 29 ++-- .../resolutions/last_value.rs | 8 +- .../data_manipulation/resolutions/sum.rs | 8 +- .../data_source/kinds/remote_db/query/one.rs | 129 +++++++++++++++++- stats/stats/src/data_source/types.rs | 6 +- 9 files changed, 173 insertions(+), 42 deletions(-) diff --git a/stats/stats/Cargo.toml b/stats/stats/Cargo.toml index 52d34f84c..27ce00015 100644 --- a/stats/stats/Cargo.toml +++ b/stats/stats/Cargo.toml @@ -42,6 +42,7 @@ sea-orm = { version = "0.12", features = [ "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls", + "mock" ] } rust_decimal = "1.27" rust_decimal_macros = "1.27" diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index edcc33045..79321a396 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -104,7 +104,7 @@ mod tests { use super::*; use crate::{ charts::db_interaction::read::get_min_block_blockscout, - data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, + data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, query_dispatch::{serialize_line_points, QuerySerialized}, range::UniversalRange, tests::{ @@ -168,13 +168,13 @@ mod tests { // Note that update is not full, therefore there is no entry with date `2022-11-09` and // wrong value is kept - let mut cx = UpdateContext { + let mut cx = UpdateContext::from_params_now_or_override(UpdateParameters { db: &db, blockscout: &blockscout, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: current_time, + update_time_override: Some(current_time), force_full: false, - }; + }); NewBlocks::update_recursively(&cx).await.unwrap(); let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) .await @@ -244,13 +244,13 @@ mod tests { .await .unwrap(); - let cx = UpdateContext { + let cx = UpdateContext::from_params_now_or_override(UpdateParameters { db: &db, blockscout: &blockscout, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: current_time, + update_time_override: Some(current_time), force_full: true, - }; + }); NewBlocks::update_recursively(&cx).await.unwrap(); let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) .await @@ -341,13 +341,13 @@ mod tests { .await .unwrap(); - let cx = UpdateContext { + let cx = UpdateContext::from_params_now_or_override(UpdateParameters { db: &db, blockscout: &blockscout, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: current_time, + update_time_override: Some(current_time), force_full: false, - }; + }); NewBlocks::update_recursively(&cx).await.unwrap(); let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) .await diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs index 9cf11d312..3e6d68786 100644 --- a/stats/stats/src/charts/query_dispatch.rs +++ b/stats/stats/src/charts/query_dispatch.rs @@ -35,7 +35,7 @@ pub trait QuerySerialized { /// Retrieve chart data from local storage. fn query_data<'a>( &self, - cx: &UpdateContext<'a>, + cx: &'a UpdateContext<'a>, range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, @@ -43,7 +43,7 @@ pub trait QuerySerialized { /// Retrieve chart data from local storage. fn query_data_static<'a>( - cx: &UpdateContext<'a>, + cx: &'a UpdateContext<'a>, range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, @@ -151,7 +151,7 @@ where fn query_data<'a>( &self, - cx: &UpdateContext<'a>, + cx: &'a UpdateContext<'a>, range: UniversalRange>, points_limit: Option, fill_missing_dates: bool, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs index a7b4581b7..a75438808 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs @@ -83,7 +83,7 @@ mod tests { use std::sync::Arc; use crate::{ - data_source::types::BlockscoutMigrations, + data_source::{types::BlockscoutMigrations, UpdateParameters}, gettable_const, lines::PredefinedMockSource, range::UniversalRange, @@ -160,13 +160,13 @@ mod tests { sea_orm::Database::connect("sqlite::memory:").await.unwrap(), )); - let context = UpdateContext { + let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, blockscout: &empty_db, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: dt("2024-07-30T09:00:00").and_utc(), + update_time_override: Some(dt("2024-07-30T09:00:00").and_utc()), force_full: false, - }; + }); assert_eq!( ::query_data( &context, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs index d67993561..800c348f0 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs @@ -140,7 +140,10 @@ mod tests { use std::{ops::Range, sync::Arc}; use crate::{ - data_source::{kinds::data_manipulation::map::MapParseTo, types::BlockscoutMigrations}, + data_source::{ + kinds::data_manipulation::map::MapParseTo, types::BlockscoutMigrations, + UpdateParameters, + }, gettable_const, lines::{PredefinedMockSource, PseudoRandomMockRetrieve}, tests::point_construction::{d, d_v_double, d_v_int, dt, w_v_double, week_of}, @@ -204,13 +207,13 @@ mod tests { sea_orm::Database::connect("sqlite::memory:").await.unwrap(), )); let output: Vec> = TestedAverageSource::query_data( - &UpdateContext { + &UpdateContext::from_params_now_or_override(UpdateParameters { db: &db, blockscout: &db, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: dt("2024-07-15T09:00:00").and_utc(), + update_time_override: Some(dt("2024-07-15T09:00:00").and_utc()), force_full: false, - }, + }), (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc()).into(), &mut AggregateTimer::new(), ) @@ -254,13 +257,13 @@ mod tests { sea_orm::Database::connect("sqlite::memory:").await.unwrap(), )); - let context = UpdateContext { + let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, blockscout: &empty_db, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: dt("2024-07-30T09:00:00").and_utc(), + update_time_override: Some(dt("2024-07-30T09:00:00").and_utc()), force_full: false, - }; + }); let week_1_average = (5.0 * 100.0 + 34.2 * 2.0 + 10.3 * 12.0) / (100.0 + 2.0 + 12.0); assert_eq!( TestedAverageSource::query_data( @@ -306,13 +309,13 @@ mod tests { sea_orm::Database::connect("sqlite::memory:").await.unwrap(), )); - let context = UpdateContext { + let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, blockscout: &empty_db, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: dt("2023-03-30T09:00:00").and_utc(), + update_time_override: Some(dt("2023-03-30T09:00:00").and_utc()), force_full: false, - }; + }); assert_eq!( TestedAverageSource::query_data( &context, @@ -354,13 +357,13 @@ mod tests { sea_orm::Database::connect("sqlite::memory:").await.unwrap(), )); - let context = UpdateContext { + let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, blockscout: &empty_db, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: dt("2023-03-30T09:00:00").and_utc(), + update_time_override: Some(dt("2023-03-30T09:00:00").and_utc()), force_full: false, - }; + }); assert_eq!( TestedAverageSource::query_data( &context, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs index 8e14e9c74..4f1549fbe 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs @@ -83,7 +83,7 @@ mod tests { use pretty_assertions::assert_eq; use crate::{ - data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, + data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, gettable_const, lines::PredefinedMockSource, range::UniversalRange, @@ -116,13 +116,13 @@ mod tests { sea_orm::Database::connect("sqlite::memory:").await.unwrap(), )); - let context = UpdateContext { + let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, blockscout: &empty_db, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: dt("2024-07-30T09:00:00").and_utc(), + update_time_override: Some(dt("2024-07-30T09:00:00").and_utc()), force_full: false, - }; + }); assert_eq!( MockSource::query_data(&context, UniversalRange::full(), &mut AggregateTimer::new()) .await diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs index c6939d1d6..a50fc6cfd 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs @@ -103,7 +103,7 @@ mod tests { use pretty_assertions::assert_eq; use crate::{ - data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, + data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, gettable_const, lines::PredefinedMockSource, range::UniversalRange, @@ -136,13 +136,13 @@ mod tests { sea_orm::Database::connect("sqlite::memory:").await.unwrap(), )); - let context = UpdateContext { + let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, blockscout: &empty_db, blockscout_applied_migrations: BlockscoutMigrations::latest(), - time: dt("2024-07-30T09:00:00").and_utc(), + update_time_override: Some(dt("2024-07-30T09:00:00").and_utc()), force_full: false, - }; + }); assert_eq!( MockSource::query_data(&context, UniversalRange::full(), &mut AggregateTimer::new()) .await diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index c80a4113c..b2c1e0de5 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -6,7 +6,7 @@ use sea_orm::{FromQueryResult, Statement, TryGetable}; use crate::{ data_source::{ kinds::remote_db::RemoteQueryBehaviour, - types::{BlockscoutMigrations, UpdateContext}, + types::{BlockscoutMigrations, Cacheable, UpdateContext}, }, range::{inclusive_range_to_exclusive, UniversalRange}, types::TimespanValue, @@ -101,3 +101,130 @@ where }) } } + +/// Will reuse result for the same produced query within one update +/// (based on update context) +pub struct PullOne24hCached(PhantomData<(S, Value)>) +where + S: StatementFromRange, + Value: Send + TryGetable; + +impl RemoteQueryBehaviour for PullOne24hCached +where + S: StatementFromRange, + Value: Send + TryGetable + Cacheable + Clone, +{ + type Output = TimespanValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result, ChartError> { + let update_time = cx.time; + let range_24h = update_time + .checked_sub_signed(TimeDelta::hours(24)) + .unwrap_or(DateTime::::MIN_UTC)..=update_time; + let query = S::get_statement( + Some(inclusive_range_to_exclusive(range_24h)), + &cx.blockscout_applied_migrations, + ); + + let value = if let Some(cached) = cx.cache.get::(&query).await { + cached + } else { + let value = FromQueryValue::::find_by_statement(query.clone()) + .one(cx.blockscout.connection.as_ref()) + .await + .map_err(ChartError::BlockscoutDB)? + .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; + let v = value.value; + cx.cache.insert(&query, v.clone()).await; + v + }; + + Ok(TimespanValue { + timespan: update_time.date_naive(), + value, + }) + } +} + +#[cfg(test)] +mod test { + use std::{collections::BTreeMap, ops::Range, sync::Arc}; + + use chrono::{DateTime, Utc}; + use pretty_assertions::assert_eq; + use sea_orm::{DatabaseBackend, DbBackend, MockDatabase, Statement}; + + use crate::{ + data_source::{ + kinds::remote_db::{RemoteQueryBehaviour, StatementFromRange}, + types::BlockscoutMigrations, + UpdateContext, UpdateParameters, + }, + range::UniversalRange, + tests::point_construction::dt, + types::TimespanValue, + utils::MarkedDbConnection, + }; + + use super::PullOne24hCached; + + struct TestStatement; + impl StatementFromRange for TestStatement { + fn get_statement( + _range: Option>>, + _completed_migrations: &BlockscoutMigrations, + ) -> Statement { + Statement::from_string(DbBackend::Postgres, "SELECT id as value FROM t;") + } + } + + type TestedPullCached = PullOne24hCached; + + #[tokio::test] + async fn pull_caching_works() { + let expected = "value1".to_string(); + let mock_db = MockDatabase::new(DatabaseBackend::Postgres) + .append_query_results([ + // First query result + vec![BTreeMap::from([( + "value", + sea_orm::Value::from(expected.clone()), + )])], + // Second query result + vec![BTreeMap::from([("value", sea_orm::Value::from("value2"))])], + ]) + .into_connection(); + let db = MarkedDbConnection::main_connection(Arc::new(mock_db)); + + let time = dt("2023-01-01T00:00:00").and_utc(); + let cx = UpdateContext::from_params_now_or_override(UpdateParameters { + db: &db, + blockscout: &db, + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: Some(time), + force_full: false, + }); + assert_eq!( + TimespanValue { + timespan: time.date_naive(), + value: expected.clone() + }, + TestedPullCached::query_data(&cx, UniversalRange::full()) + .await + .unwrap() + ); + // the result is cached + assert_eq!( + TimespanValue { + timespan: time.date_naive(), + value: expected.clone() + }, + TestedPullCached::query_data(&cx, UniversalRange::full()) + .await + .unwrap() + ); + } +} diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 8f226970e..878e08aa5 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -200,7 +200,7 @@ impl UpdateCache { /// If the cache did not have value for this query present, None is returned. /// /// If the cache did have this query present, the value is updated, and the old value is returned. - async fn insert(&mut self, query: &Statement, value: V) -> Option { + pub async fn insert(&self, query: &Statement, value: V) -> Option { self.inner .lock() .await @@ -209,7 +209,7 @@ impl UpdateCache { } /// Returns a value for this query, if present - async fn get(&mut self, query: &Statement) -> Option { + pub async fn get(&self, query: &Statement) -> Option { self.inner .lock() .await @@ -255,7 +255,7 @@ mod tests { #[tokio::test] async fn cache_works() { - let mut cache = UpdateCache::new(); + let cache = UpdateCache::new(); let stmt_a = Statement::from_string(DbBackend::Sqlite, "abcde"); let stmt_b = Statement::from_string(DbBackend::Sqlite, "edcba"); From c4ecddd60d7ccfc16ebbf468f3d5b1c4589e86a7 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 20 Dec 2024 00:04:15 +0900 Subject: [PATCH 11/17] migrate all 24 hours charts to single sql statement + cache --- .../charts/counters/average_txn_fee_24h.rs | 150 +++++++++--------- stats/stats/src/charts/counters/mod.rs | 1 + .../stats/src/charts/counters/new_txns_24h.rs | 77 +++------ .../stats/src/charts/counters/txns_fee_24h.rs | 107 +++---------- .../src/data_source/kinds/remote_db/mod.rs | 4 +- .../data_source/kinds/remote_db/query/mod.rs | 2 +- .../data_source/kinds/remote_db/query/one.rs | 38 +++-- stats/stats/src/data_source/types.rs | 39 +++-- 8 files changed, 173 insertions(+), 245 deletions(-) diff --git a/stats/stats/src/charts/counters/average_txn_fee_24h.rs b/stats/stats/src/charts/counters/average_txn_fee_24h.rs index 171bd08e1..e371109d1 100644 --- a/stats/stats/src/charts/counters/average_txn_fee_24h.rs +++ b/stats/stats/src/charts/counters/average_txn_fee_24h.rs @@ -3,96 +3,102 @@ use std::ops::Range; use crate::{ data_source::{ kinds::{ - data_manipulation::map::{MapToString, UnwrapOr}, + data_manipulation::map::{Map, MapFunction, MapToString, UnwrapOr}, local_db::DirectPointLocalDbChartSource, - remote_db::{PullOne24h, RemoteDatabaseSource, StatementFromRange}, + remote_db::{PullOne24hCached, RemoteDatabaseSource, StatementFromRange}, }, types::BlockscoutMigrations, }, gettable_const, - utils::{produce_filter_and_values, sql_with_range_filter_opt}, + types::TimespanValue, ChartProperties, MissingDatePolicy, Named, }; +use blockscout_db::entity::{blocks, transactions}; use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{DbBackend, Statement}; +use migration::{Alias, Func}; +use sea_orm::{DbBackend, FromQueryResult, IntoSimpleExpr, QuerySelect, QueryTrait, Statement}; const ETHER: i64 = i64::pow(10, 18); -/// `AverageTxnFeeStatement` but without `group by` -pub struct AverageTxnFeeUngroupedStatement; +pub struct TxnsStatsStatement; -impl StatementFromRange for AverageTxnFeeUngroupedStatement { +impl StatementFromRange for TxnsStatsStatement { fn get_statement( range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { - if completed_migrations.denormalization { - // TODO: consider supporting such case in macro ? - let mut args = vec![ETHER.into()]; - let (tx_filter, new_args) = - produce_filter_and_values(range.clone(), "t.block_timestamp", args.len() + 1); - args.extend(new_args); - let (block_filter, new_args) = - produce_filter_and_values(range.clone(), "b.timestamp", args.len() + 1); - args.extend(new_args); - let sql = format!( - r#" - SELECT - (AVG( - t_filtered.gas_used * - COALESCE( - t_filtered.gas_price, - b.base_fee_per_gas + LEAST( - t_filtered.max_priority_fee_per_gas, - t_filtered.max_fee_per_gas - b.base_fee_per_gas - ) - ) - ) / $1)::FLOAT as value - FROM ( - SELECT * from transactions t - WHERE - t.block_consensus = true AND - t.block_timestamp != to_timestamp(0) {tx_filter} - ) as t_filtered - JOIN blocks b ON t_filtered.block_hash = b.hash - WHERE - b.timestamp != to_timestamp(0) AND - b.consensus = true {block_filter} - "#, - ); - Statement::from_sql_and_values(DbBackend::Postgres, sql, args) - } else { - sql_with_range_filter_opt!( - DbBackend::Postgres, - r#" - SELECT - (AVG( - t.gas_used * - COALESCE( - t.gas_price, - b.base_fee_per_gas + LEAST( - t.max_priority_fee_per_gas, - t.max_fee_per_gas - b.base_fee_per_gas - ) - ) - ) / $1)::FLOAT as value - FROM transactions t - JOIN blocks b ON t.block_hash = b.hash - WHERE - b.timestamp != to_timestamp(0) AND - b.consensus = true {filter} - "#, - [ETHER.into()], - "b.timestamp", - range - ) + use sea_orm::prelude::*; + + let fee_query = Expr::cust_with_exprs( + "COALESCE($1, $2 + LEAST($3, $4))", + [ + transactions::Column::GasPrice.into_simple_expr(), + blocks::Column::BaseFeePerGas.into_simple_expr(), + transactions::Column::MaxPriorityFeePerGas.into_simple_expr(), + transactions::Column::MaxFeePerGas + .into_simple_expr() + .sub(blocks::Column::BaseFeePerGas.into_simple_expr()), + ], + ) + .mul(transactions::Column::GasUsed.into_simple_expr()); + + let count_query = Func::count(transactions::Column::Hash.into_simple_expr()); + let sum_query = Expr::expr(Func::sum(fee_query.clone())) + .div(ETHER) + .cast_as(Alias::new("FLOAT")); + let avg_query = Expr::expr(Func::avg(fee_query)) + .div(ETHER) + .cast_as(Alias::new("FLOAT")); + + let base_query = blocks::Entity::find().inner_join(transactions::Entity); + let mut query = base_query + .select_only() + .expr_as(count_query, "count") + .expr_as(sum_query, "fee_sum") + .expr_as(avg_query, "fee_average") + .to_owned(); + + if let Some(r) = range { + if completed_migrations.denormalization { + query = query + .filter(transactions::Column::BlockTimestamp.lt(r.end)) + .filter(transactions::Column::BlockTimestamp.gte(r.start)) + } + query = query + .filter(blocks::Column::Timestamp.lt(r.end)) + .filter(blocks::Column::Timestamp.gte(r.start)); } + + query.build(DbBackend::Postgres) + } +} + +#[derive(Debug, Clone, FromQueryResult)] +pub struct TxnsStatsValue { + pub count: i64, + pub fee_average: Option, + pub fee_sum: Option, +} + +pub type Txns24hStats = RemoteDatabaseSource>; + +pub struct ExtractAverage; + +impl MapFunction> for ExtractAverage { + type Output = TimespanValue>; + + fn function( + inner_data: TimespanValue, + ) -> Result { + Ok(TimespanValue { + timespan: inner_data.timespan, + value: inner_data.value.fee_average, + }) } } -pub type AverageTxnFee24hRemote = - RemoteDatabaseSource>>; +pub type AverageTxnFee24hExtracted = Map; pub struct Properties; @@ -115,8 +121,10 @@ impl ChartProperties for Properties { gettable_const!(Zero: f64 = 0.0); -pub type AverageTxnFee24h = - DirectPointLocalDbChartSource>, Properties>; +pub type AverageTxnFee24h = DirectPointLocalDbChartSource< + MapToString>, + Properties, +>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/mod.rs b/stats/stats/src/charts/counters/mod.rs index 37521eb34..f66b23bb1 100644 --- a/stats/stats/src/charts/counters/mod.rs +++ b/stats/stats/src/charts/counters/mod.rs @@ -23,6 +23,7 @@ mod mock; pub use average_block_time::AverageBlockTime; pub use average_txn_fee_24h::AverageTxnFee24h; +pub(crate) use average_txn_fee_24h::TxnsStatsValue; pub use completed_txns::CompletedTxns; pub use last_new_contracts::LastNewContracts; pub use last_new_verified_contracts::LastNewVerifiedContracts; diff --git a/stats/stats/src/charts/counters/new_txns_24h.rs b/stats/stats/src/charts/counters/new_txns_24h.rs index a6aecbfcb..ad8397a42 100644 --- a/stats/stats/src/charts/counters/new_txns_24h.rs +++ b/stats/stats/src/charts/counters/new_txns_24h.rs @@ -1,67 +1,32 @@ -use std::ops::Range; - use crate::{ - data_source::{ - kinds::{ - local_db::DirectPointLocalDbChartSource, - remote_db::{ - PullOne24h, RemoteDatabaseSource, StatementFromRange, - }, - }, - types::BlockscoutMigrations, + data_source::kinds::{ + data_manipulation::map::{Map, MapFunction}, + local_db::DirectPointLocalDbChartSource, }, - utils::sql_with_range_filter_opt, ChartProperties, MissingDatePolicy, Named, + types::TimespanValue, + ChartError, ChartProperties, MissingDatePolicy, Named, }; -use chrono::{DateTime, NaiveDate, Utc}; +use chrono::NaiveDate; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{DbBackend, Statement}; -/// `NewTxnsStatement` but without `group by` -pub struct NewTxnsUngroupedStatement; +use super::{average_txn_fee_24h::Txns24hStats, TxnsStatsValue}; + +pub struct ExtractCount; -impl StatementFromRange for NewTxnsUngroupedStatement { - fn get_statement( - range: Option>>, - completed_migrations: &BlockscoutMigrations, - ) -> Statement { - // do not filter by `!= to_timestamp(0)` because - // 1. it allows to use index `transactions_block_consensus_index` - // 2. there is no reason not to count genesis transactions - if completed_migrations.denormalization { - sql_with_range_filter_opt!( - DbBackend::Postgres, - r#" - SELECT - COUNT(*)::TEXT as value - FROM transactions t - WHERE - t.block_consensus = true {filter}; - "#, - [], - "t.block_timestamp", - range - ) - } else { - sql_with_range_filter_opt!( - DbBackend::Postgres, - r#" - SELECT - COUNT(*)::TEXT as value - FROM transactions t - JOIN blocks b ON t.block_hash = b.hash - WHERE - b.consensus = true {filter}; - "#, - [], - "b.timestamp", - range - ) - } +impl MapFunction> for ExtractCount { + type Output = TimespanValue; + + fn function( + inner_data: TimespanValue, + ) -> Result { + Ok(TimespanValue { + timespan: inner_data.timespan, + value: inner_data.value.count.to_string(), + }) } } -pub type NewTxns24hRemote = RemoteDatabaseSource>; - +pub type NewTxns24hExtracted = Map; pub struct Properties; impl Named for Properties { @@ -81,7 +46,7 @@ impl ChartProperties for Properties { } } -pub type NewTxns24h = DirectPointLocalDbChartSource; +pub type NewTxns24h = DirectPointLocalDbChartSource; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/txns_fee_24h.rs b/stats/stats/src/charts/counters/txns_fee_24h.rs index f74194129..c09bc0a12 100644 --- a/stats/stats/src/charts/counters/txns_fee_24h.rs +++ b/stats/stats/src/charts/counters/txns_fee_24h.rs @@ -1,97 +1,34 @@ -use std::ops::Range; use crate::{ - data_source::{ - kinds::{ - data_manipulation::map::{MapToString, UnwrapOr}, + data_source::kinds::{ + data_manipulation::map::{Map, MapFunction, MapToString, UnwrapOr}, local_db::DirectPointLocalDbChartSource, - remote_db::{PullOne24h, RemoteDatabaseSource, StatementFromRange}, }, - types::BlockscoutMigrations, - }, gettable_const, - utils::{produce_filter_and_values, sql_with_range_filter_opt}, - ChartProperties, MissingDatePolicy, Named, + types::TimespanValue, + ChartError, ChartProperties, MissingDatePolicy, Named, }; -use chrono::{DateTime, NaiveDate, Utc}; +use chrono::NaiveDate; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{DbBackend, Statement}; - -const ETHER: i64 = i64::pow(10, 18); - -pub struct TxnsFeeUngroupedStatement; - -impl StatementFromRange for TxnsFeeUngroupedStatement { - fn get_statement( - range: Option>>, - completed_migrations: &BlockscoutMigrations, - ) -> Statement { - if completed_migrations.denormalization { - // TODO: consider supporting such case in macro ? - let mut args = vec![ETHER.into()]; - let (tx_filter, new_args) = - produce_filter_and_values(range.clone(), "t.block_timestamp", args.len() + 1); - args.extend(new_args); - let (block_filter, new_args) = - produce_filter_and_values(range.clone(), "b.timestamp", args.len() + 1); - args.extend(new_args); - let sql = format!( - r#" - SELECT - (SUM( - t_filtered.gas_used * - COALESCE( - t_filtered.gas_price, - b.base_fee_per_gas + LEAST( - t_filtered.max_priority_fee_per_gas, - t_filtered.max_fee_per_gas - b.base_fee_per_gas - ) - ) - ) / $1)::FLOAT as value - FROM ( - SELECT * from transactions t - WHERE - t.block_consensus = true AND - t.block_timestamp != to_timestamp(0) {tx_filter} - ) as t_filtered - JOIN blocks b ON t_filtered.block_hash = b.hash - WHERE - b.timestamp != to_timestamp(0) AND - b.consensus = true {block_filter} - "#, - ); - Statement::from_sql_and_values(DbBackend::Postgres, sql, args) - } else { - sql_with_range_filter_opt!( - DbBackend::Postgres, - r#" - SELECT - (SUM( - t.gas_used * - COALESCE( - t.gas_price, - b.base_fee_per_gas + LEAST( - t.max_priority_fee_per_gas, - t.max_fee_per_gas - b.base_fee_per_gas - ) - ) - ) / $1)::FLOAT as value - FROM transactions t - JOIN blocks b ON t.block_hash = b.hash - WHERE - b.timestamp != to_timestamp(0) AND - b.consensus = true {filter} - "#, - [ETHER.into()], - "b.timestamp", - range - ) - } + +use super::{average_txn_fee_24h::Txns24hStats, TxnsStatsValue}; + +pub struct ExtractSum; + +impl MapFunction> for ExtractSum { + type Output = TimespanValue>; + + fn function( + inner_data: TimespanValue, + ) -> Result { + Ok(TimespanValue { + timespan: inner_data.timespan, + value: inner_data.value.fee_sum, + }) } } -pub type TxnsFee24hRemote = - RemoteDatabaseSource>>; +pub type TxnsFee24hExtracted = Map; pub struct Properties; @@ -115,7 +52,7 @@ impl ChartProperties for Properties { gettable_const!(Zero: f64 = 0.0); pub type TxnsFee24h = - DirectPointLocalDbChartSource>, Properties>; + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/data_source/kinds/remote_db/mod.rs b/stats/stats/src/data_source/kinds/remote_db/mod.rs index e326c5ce9..4e01af8bc 100644 --- a/stats/stats/src/data_source/kinds/remote_db/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/mod.rs @@ -34,8 +34,8 @@ use crate::{ }; pub use query::{ - PullAllWithAndSort, PullEachWith, PullOne, PullOne24h, StatementForOne, StatementFromRange, - StatementFromTimespan, + PullAllWithAndSort, PullEachWith, PullOne, PullOne24h, PullOne24hCached, StatementForOne, + StatementFromRange, StatementFromTimespan, }; /// See [module-level documentation](self) diff --git a/stats/stats/src/data_source/kinds/remote_db/query/mod.rs b/stats/stats/src/data_source/kinds/remote_db/query/mod.rs index 9a3b114a7..9ddeb3a58 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/mod.rs @@ -4,4 +4,4 @@ mod one; pub use all::{PullAllWithAndSort, StatementFromRange}; pub use each::{PullEachWith, StatementFromTimespan}; -pub use one::{PullOne, PullOne24h, StatementForOne}; +pub use one::{PullOne, PullOne24h, PullOne24hCached, StatementForOne}; diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index b2c1e0de5..5d4a690ef 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -1,7 +1,7 @@ use std::marker::{PhantomData, Send}; use chrono::{DateTime, NaiveDate, TimeDelta, Utc}; -use sea_orm::{FromQueryResult, Statement, TryGetable}; +use sea_orm::{FromQueryResult, Statement}; use crate::{ data_source::{ @@ -55,11 +55,7 @@ where } } -#[derive(FromQueryResult)] -struct FromQueryValue { - value: V, -} - +// todo: remove unused /// Retrieve a single value `Value` for the period of last 24h /// according to provided statemnt `S`. /// @@ -67,12 +63,12 @@ struct FromQueryValue { pub struct PullOne24h(PhantomData<(S, Value)>) where S: StatementFromRange, - Value: Send + TryGetable; + Value: FromQueryResult + Send; impl RemoteQueryBehaviour for PullOne24h where S: StatementFromRange, - Value: Send + TryGetable, + Value: FromQueryResult + Send, { type Output = TimespanValue; @@ -89,7 +85,7 @@ where &cx.blockscout_applied_migrations, ); - let value = FromQueryValue::::find_by_statement(query) + let value = Value::find_by_statement(query) .one(cx.blockscout.connection.as_ref()) .await .map_err(ChartError::BlockscoutDB)? @@ -97,7 +93,7 @@ where Ok(TimespanValue { timespan: update_time.date_naive(), - value: value.value, + value, }) } } @@ -107,12 +103,12 @@ where pub struct PullOne24hCached(PhantomData<(S, Value)>) where S: StatementFromRange, - Value: Send + TryGetable; + Value: FromQueryResult + Cacheable + Clone + Send; impl RemoteQueryBehaviour for PullOne24hCached where S: StatementFromRange, - Value: Send + TryGetable + Cacheable + Clone, + Value: FromQueryResult + Cacheable + Clone + Send, { type Output = TimespanValue; @@ -132,14 +128,14 @@ where let value = if let Some(cached) = cx.cache.get::(&query).await { cached } else { - let value = FromQueryValue::::find_by_statement(query.clone()) + let find_by_statement = Value::find_by_statement(query.clone()); + let value = find_by_statement .one(cx.blockscout.connection.as_ref()) .await .map_err(ChartError::BlockscoutDB)? .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; - let v = value.value; - cx.cache.insert(&query, v.clone()).await; - v + cx.cache.insert(&query, value.clone()).await; + value }; Ok(TimespanValue { @@ -160,7 +156,7 @@ mod test { use crate::{ data_source::{ kinds::remote_db::{RemoteQueryBehaviour, StatementFromRange}, - types::BlockscoutMigrations, + types::{BlockscoutMigrations, WrappedValue}, UpdateContext, UpdateParameters, }, range::UniversalRange, @@ -181,17 +177,19 @@ mod test { } } - type TestedPullCached = PullOne24hCached; + type TestedPullCached = PullOne24hCached>; #[tokio::test] async fn pull_caching_works() { - let expected = "value1".to_string(); + let expected = WrappedValue { + value: "value1".to_string(), + }; let mock_db = MockDatabase::new(DatabaseBackend::Postgres) .append_query_results([ // First query result vec![BTreeMap::from([( "value", - sea_orm::Value::from(expected.clone()), + sea_orm::Value::from(expected.value.clone()), )])], // Second query result vec![BTreeMap::from([("value", sea_orm::Value::from("value2"))])], diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 878e08aa5..0ec5c6f1c 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -2,11 +2,13 @@ use std::{collections::HashMap, sync::Arc}; use blockscout_db::entity::migrations_status; use chrono::Utc; -use sea_orm::{DatabaseConnection, DbErr, EntityTrait, FromQueryResult, QueryOrder, Statement}; +use sea_orm::{ + DatabaseConnection, DbErr, EntityTrait, FromQueryResult, QueryOrder, Statement, TryGetable, +}; use tokio::sync::Mutex; use tracing::warn; -use crate::utils::MarkedDbConnection; +use crate::{counters::TxnsStatsValue, utils::MarkedDbConnection}; #[derive(Clone)] pub struct UpdateParameters<'a> { @@ -138,13 +140,11 @@ impl BlockscoutMigrations { pub enum CacheValue { ValueString(String), ValueOptionF64(Option), + ValueTxnsStats(TxnsStatsValue), } pub trait Cacheable { fn from_entry(entry: CacheValue) -> Option - where - Self: Sized; - fn from_entry_ref(entry: &CacheValue) -> Option<&Self> where Self: Sized; fn into_entry(self) -> CacheValue; @@ -163,25 +163,44 @@ macro_rules! impl_cacheable { } } - fn from_entry_ref(entry: &CacheValue) -> Option<&Self> + fn into_entry(self) -> CacheValue { + CacheValue::$cache_value_variant(self) + } + } + }; +} + +impl_cacheable!(String, ValueString); +impl_cacheable!(Option, ValueOptionF64); +impl_cacheable!(TxnsStatsValue, ValueTxnsStats); + +#[derive(Debug, Clone, FromQueryResult, PartialEq, Eq, PartialOrd, Ord)] +pub struct WrappedValue { + pub value: V, +} + +macro_rules! impl_cacheable_wrapped { + ($type: ty, $cache_value_variant:ident) => { + impl Cacheable for $type { + fn from_entry(entry: CacheValue) -> Option where Self: Sized, { match entry { - CacheValue::$cache_value_variant(s) => Some(s), + CacheValue::$cache_value_variant(s) => Some(WrappedValue { value: s }), _ => None, } } fn into_entry(self) -> CacheValue { - CacheValue::$cache_value_variant(self) + CacheValue::$cache_value_variant(self.value) } } }; } -impl_cacheable!(String, ValueString); -impl_cacheable!(Option, ValueOptionF64); +impl_cacheable_wrapped!(WrappedValue, ValueString); +impl_cacheable_wrapped!(WrappedValue>, ValueOptionF64); #[derive(Clone, Debug)] pub struct UpdateCache { From cd67595615d17e2e36c04cd39b701aebce8516bd Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 20 Dec 2024 00:11:27 +0900 Subject: [PATCH 12/17] remove unused query behaviour --- .../src/data_source/kinds/remote_db/mod.rs | 2 +- .../data_source/kinds/remote_db/query/mod.rs | 2 +- .../data_source/kinds/remote_db/query/one.rs | 43 ------------------- 3 files changed, 2 insertions(+), 45 deletions(-) diff --git a/stats/stats/src/data_source/kinds/remote_db/mod.rs b/stats/stats/src/data_source/kinds/remote_db/mod.rs index 4e01af8bc..0fbdb4b56 100644 --- a/stats/stats/src/data_source/kinds/remote_db/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/mod.rs @@ -34,7 +34,7 @@ use crate::{ }; pub use query::{ - PullAllWithAndSort, PullEachWith, PullOne, PullOne24h, PullOne24hCached, StatementForOne, + PullAllWithAndSort, PullEachWith, PullOne, PullOne24hCached, StatementForOne, StatementFromRange, StatementFromTimespan, }; diff --git a/stats/stats/src/data_source/kinds/remote_db/query/mod.rs b/stats/stats/src/data_source/kinds/remote_db/query/mod.rs index 9ddeb3a58..cad537e1c 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/mod.rs @@ -4,4 +4,4 @@ mod one; pub use all::{PullAllWithAndSort, StatementFromRange}; pub use each::{PullEachWith, StatementFromTimespan}; -pub use one::{PullOne, PullOne24h, PullOne24hCached, StatementForOne}; +pub use one::{PullOne, PullOne24hCached, StatementForOne}; diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index 5d4a690ef..9be5b0798 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -55,49 +55,6 @@ where } } -// todo: remove unused -/// Retrieve a single value `Value` for the period of last 24h -/// according to provided statemnt `S`. -/// -/// The point will be for the time of update -pub struct PullOne24h(PhantomData<(S, Value)>) -where - S: StatementFromRange, - Value: FromQueryResult + Send; - -impl RemoteQueryBehaviour for PullOne24h -where - S: StatementFromRange, - Value: FromQueryResult + Send, -{ - type Output = TimespanValue; - - async fn query_data( - cx: &UpdateContext<'_>, - _range: UniversalRange>, - ) -> Result, ChartError> { - let update_time = cx.time; - let range_24h = update_time - .checked_sub_signed(TimeDelta::hours(24)) - .unwrap_or(DateTime::::MIN_UTC)..=update_time; - let query = S::get_statement( - Some(inclusive_range_to_exclusive(range_24h)), - &cx.blockscout_applied_migrations, - ); - - let value = Value::find_by_statement(query) - .one(cx.blockscout.connection.as_ref()) - .await - .map_err(ChartError::BlockscoutDB)? - .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; - - Ok(TimespanValue { - timespan: update_time.date_naive(), - value, - }) - } -} - /// Will reuse result for the same produced query within one update /// (based on update context) pub struct PullOne24hCached(PhantomData<(S, Value)>) From 087bfd746bccf8de9c0e931e5e2254013da2b2bd Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 20 Dec 2024 00:30:58 +0900 Subject: [PATCH 13/17] Revert "mark db connections to avoid cache clashing" This reverts commit 4a4a95900a366131365a34ced352c82c2b00b4ae. --- stats/stats-server/src/read_service.rs | 14 +++--- stats/stats-server/src/server.rs | 12 ++--- stats/stats-server/src/update_service.rs | 26 +++++------ .../src/charts/counters/average_block_time.rs | 7 ++- .../stats/src/charts/counters/pending_txns.rs | 2 +- .../src/charts/counters/total_addresses.rs | 18 +++----- .../stats/src/charts/counters/total_blocks.rs | 46 +++++++++---------- .../src/charts/counters/total_contracts.rs | 3 +- stats/stats/src/charts/counters/total_txns.rs | 21 ++++----- .../src/charts/counters/yesterday_txns.rs | 3 +- stats/stats/src/charts/db_interaction/read.rs | 7 ++- .../lines/native_coin_holders_growth.rs | 40 ++++++---------- stats/stats/src/charts/lines/new_accounts.rs | 2 +- .../src/charts/lines/new_block_rewards.rs | 8 ++-- stats/stats/src/charts/lines/new_blocks.rs | 38 +++++++-------- .../stats/src/charts/lines/new_txns_window.rs | 7 ++- .../data_manipulation/filter_deducible.rs | 7 +-- .../data_manipulation/resolutions/average.rs | 19 ++------ .../resolutions/last_value.rs | 7 +-- .../data_manipulation/resolutions/sum.rs | 7 +-- .../src/data_source/kinds/local_db/mod.rs | 19 ++++---- .../kinds/local_db/parameters/query.rs | 17 +++---- .../parameters/update/batching/mod.rs | 11 ++--- .../kinds/local_db/parameters/update/point.rs | 2 +- .../data_source/kinds/remote_db/query/all.rs | 2 +- .../data_source/kinds/remote_db/query/each.rs | 2 +- .../data_source/kinds/remote_db/query/one.rs | 11 ++--- stats/stats/src/data_source/tests.rs | 8 ++-- stats/stats/src/data_source/types.rs | 10 ++-- stats/stats/src/tests/init_db.rs | 10 ---- stats/stats/src/tests/simple_test.rs | 41 +++++++---------- stats/stats/src/utils.rs | 41 +---------------- 32 files changed, 173 insertions(+), 295 deletions(-) diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 9ac964344..9a0af64b3 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -13,13 +13,13 @@ use async_trait::async_trait; use chrono::{DateTime, NaiveDate, Utc}; use futures::{stream::FuturesOrdered, StreamExt}; use proto_v1::stats_service_server::StatsService; -use sea_orm::DbErr; +use sea_orm::{DatabaseConnection, DbErr}; use stats::{ data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, query_dispatch::{CounterHandle, LineHandle, QuerySerializedDyn}, range::UniversalRange, types::Timespan, - utils::{day_start, MarkedDbConnection}, + utils::day_start, ChartError, RequestedPointsLimit, ResolutionKind, }; use stats_proto::blockscout::stats::v1 as proto_v1; @@ -27,16 +27,16 @@ use tonic::{Request, Response, Status}; #[derive(Clone)] pub struct ReadService { - db: MarkedDbConnection, - blockscout: MarkedDbConnection, + db: Arc, + blockscout: Arc, charts: Arc, limits: ReadLimits, } impl ReadService { pub async fn new( - db: MarkedDbConnection, - blockscout: MarkedDbConnection, + db: Arc, + blockscout: Arc, charts: Arc, limits: ReadLimits, ) -> Result { @@ -104,7 +104,7 @@ impl ReadService { points_limit: Option, query_time: DateTime, ) -> Result { - let migrations = BlockscoutMigrations::query_from_db(self.blockscout.connection.as_ref()) + let migrations = BlockscoutMigrations::query_from_db(&self.blockscout) .await .map_err(ChartError::BlockscoutDB)?; let context = UpdateContext::from_params_now_or_override(UpdateParameters { diff --git a/stats/stats-server/src/server.rs b/stats/stats-server/src/server.rs index b03c17e18..f198d1ab0 100644 --- a/stats/stats-server/src/server.rs +++ b/stats/stats-server/src/server.rs @@ -14,7 +14,7 @@ use anyhow::Context; use blockscout_endpoint_swagger::route_swagger; use blockscout_service_launcher::launcher::{self, LaunchSettings}; use sea_orm::{ConnectOptions, Database}; -use stats::{metrics, utils::MarkedDbConnection}; +use stats::metrics; use stats_proto::blockscout::stats::v1::{ health_actix::route_health, health_server::HealthServer, @@ -80,9 +80,7 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { settings.run_migrations, ) .await?; - let db = MarkedDbConnection::main_connection(Arc::new( - Database::connect(opt).await.context("stats DB")?, - )); + let db = Arc::new(Database::connect(opt).await.context("stats DB")?); let mut opt = ConnectOptions::new(settings.blockscout_db_url.clone()); opt.sqlx_logging_level(tracing::log::LevelFilter::Debug); @@ -92,9 +90,7 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { tracing::log::LevelFilter::Warn, Duration::from_secs(3600), ); - let blockscout = MarkedDbConnection::main_connection(Arc::new( - Database::connect(opt).await.context("blockscout DB")?, - )); + let blockscout = Arc::new(Database::connect(opt).await.context("blockscout DB")?); let charts = Arc::new(RuntimeSetup::new( charts_config, @@ -106,7 +102,7 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { for group_entry in charts.update_groups.values() { group_entry .group - .create_charts_with_mutexes(db.connection.as_ref(), None, &group_entry.enabled_members) + .create_charts_with_mutexes(&db, None, &group_entry.enabled_members) .await?; } diff --git a/stats/stats-server/src/update_service.rs b/stats/stats-server/src/update_service.rs index 543d820f7..8c05f9a80 100644 --- a/stats/stats-server/src/update_service.rs +++ b/stats/stats-server/src/update_service.rs @@ -1,19 +1,16 @@ use crate::runtime_setup::{RuntimeSetup, UpdateGroupEntry}; use chrono::Utc; use cron::Schedule; -use sea_orm::DbErr; -use stats::{ - data_source::types::{BlockscoutMigrations, UpdateParameters}, - utils::MarkedDbConnection, -}; +use sea_orm::{DatabaseConnection, DbErr}; +use stats::data_source::types::{BlockscoutMigrations, UpdateParameters}; use std::sync::Arc; use tokio::task::JoinHandle; const FAILED_UPDATERS_UNTIL_PANIC: u64 = 3; pub struct UpdateService { - db: MarkedDbConnection, - blockscout: MarkedDbConnection, + db: Arc, + blockscout: Arc, charts: Arc, } @@ -29,8 +26,8 @@ fn time_till_next_call(schedule: &Schedule) -> std::time::Duration { impl UpdateService { pub async fn new( - db: MarkedDbConnection, - blockscout: MarkedDbConnection, + db: Arc, + blockscout: Arc, charts: Arc, ) -> Result { Ok(Self { @@ -107,12 +104,11 @@ impl UpdateService { force_update = force_full, "updating group of charts" ); - let Ok(active_migrations) = - BlockscoutMigrations::query_from_db(self.blockscout.connection.as_ref()) - .await - .inspect_err(|err| { - tracing::error!("error during blockscout migrations detection: {:?}", err) - }) + let Ok(active_migrations) = BlockscoutMigrations::query_from_db(&self.blockscout) + .await + .inspect_err(|err| { + tracing::error!("error during blockscout migrations detection: {:?}", err) + }) else { return; }; diff --git a/stats/stats/src/charts/counters/average_block_time.rs b/stats/stats/src/charts/counters/average_block_time.rs index 3d865cc36..4305d215b 100644 --- a/stats/stats/src/charts/counters/average_block_time.rs +++ b/stats/stats/src/charts/counters/average_block_time.rs @@ -55,7 +55,7 @@ async fn query_average_block_time( ) -> Result>, ChartError> { let query = average_block_time_statement(offset); let block_timestamps = BlockTimestamp::find_by_statement(query) - .all(cx.blockscout.connection.as_ref()) + .all(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)?; Ok(calculate_average_block_time(block_timestamps)) @@ -149,7 +149,6 @@ mod tests { mock_blockscout::fill_many_blocks, simple_test::{get_counter, prepare_chart_test, simple_test_counter}, }, - utils::MarkedDbConnection, }; #[tokio::test] @@ -198,8 +197,8 @@ mod tests { }; fill_many_blocks(&blockscout, current_time.naive_utc(), &block_times).await; let mut parameters = UpdateParameters { - db: &MarkedDbConnection::from_test_db(&db).unwrap(), - blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), + db: &db, + blockscout: &blockscout, blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: true, diff --git a/stats/stats/src/charts/counters/pending_txns.rs b/stats/stats/src/charts/counters/pending_txns.rs index 51bce5ff0..38b305f83 100644 --- a/stats/stats/src/charts/counters/pending_txns.rs +++ b/stats/stats/src/charts/counters/pending_txns.rs @@ -53,7 +53,7 @@ impl RemoteQueryBehaviour for PendingTxnsQuery { .unwrap_or(DateTime::::MIN_UTC), ); let data = Value::find_by_statement(query) - .one(cx.blockscout.connection.as_ref()) + .one(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)? .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; diff --git a/stats/stats/src/charts/counters/total_addresses.rs b/stats/stats/src/charts/counters/total_addresses.rs index 713ce9c39..b196ade91 100644 --- a/stats/stats/src/charts/counters/total_addresses.rs +++ b/stats/stats/src/charts/counters/total_addresses.rs @@ -8,13 +8,12 @@ use crate::{ types::BlockscoutMigrations, }, types::timespans::DateValue, - utils::MarkedDbConnection, ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::addresses; use chrono::{NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{DbBackend, EntityName, Statement}; +use sea_orm::{DatabaseConnection, DbBackend, EntityName, Statement}; pub struct TotalAddressesStatement; @@ -64,18 +63,15 @@ impl ChartProperties for Properties { pub struct TotalAddressesEstimation; impl ValueEstimation for TotalAddressesEstimation { - async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError> { + async fn estimate(blockscout: &DatabaseConnection) -> Result, ChartError> { // `now()` is more relevant when taken right before the query rather than // `cx.time` measured a bit earlier. let now = Utc::now(); - let value = query_estimated_table_rows( - blockscout.connection.as_ref(), - addresses::Entity.table_name(), - ) - .await - .map_err(ChartError::BlockscoutDB)? - .map(|n| u64::try_from(n).unwrap_or(0)) - .unwrap_or(0); + let value = query_estimated_table_rows(blockscout, addresses::Entity.table_name()) + .await + .map_err(ChartError::BlockscoutDB)? + .map(|n| u64::try_from(n).unwrap_or(0)) + .unwrap_or(0); Ok(DateValue { timespan: now.date_naive(), value: value.to_string(), diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index ad1cab55f..ce8f33737 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -10,7 +10,6 @@ use crate::{ }, range::UniversalRange, types::timespans::DateValue, - utils::MarkedDbConnection, ChartError, ChartProperties, MissingDatePolicy, Named, }; @@ -40,7 +39,7 @@ impl RemoteQueryBehaviour for TotalBlocksQueryBehaviour { .column_as(Expr::col(blocks::Column::Timestamp).max(), "timestamp") .filter(blocks::Column::Consensus.eq(true)) .into_model::() - .one(cx.blockscout.connection.as_ref()) + .one(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)? .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; @@ -77,19 +76,18 @@ impl ChartProperties for Properties { pub struct TotalBlocksEstimation; impl ValueEstimation for TotalBlocksEstimation { - async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError> { + async fn estimate(blockscout: &DatabaseConnection) -> Result, ChartError> { // `now()` is more relevant when taken right before the query rather than // `cx.time` measured a bit earlier. let now = Utc::now(); - let value = - query_estimated_table_rows(blockscout.connection.as_ref(), blocks::Entity.table_name()) - .await - .map_err(ChartError::BlockscoutDB)? - .map(|b| { - let b = b as f64 * 0.9; - b as i64 - }) - .unwrap_or(0); + let value = query_estimated_table_rows(blockscout, blocks::Entity.table_name()) + .await + .map_err(ChartError::BlockscoutDB)? + .map(|b| { + let b = b as f64 * 0.9; + b as i64 + }) + .unwrap_or(0); Ok(DateValue { timespan: now.date_naive(), value: value.to_string(), @@ -107,7 +105,7 @@ mod tests { use crate::{ data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, tests::{ - init_db::init_marked_db_all, + init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data, simple_test::{get_counter, test_counter_fallback}, }, @@ -122,11 +120,11 @@ mod tests { #[ignore = "needs database to run"] async fn update_total_blocks_recurrent() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all("update_total_blocks_recurrent").await; + let (db, blockscout) = init_db_all("update_total_blocks_recurrent").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db.connection, ¤t_time) + TotalBlocks::init_recursively(&db, ¤t_time) .await .unwrap(); @@ -136,11 +134,11 @@ mod tests { value: Set(1.to_string()), ..Default::default() }) - .exec(&db.connection as &DatabaseConnection) + .exec(&db as &DatabaseConnection) .await .unwrap(); - fill_mock_blockscout_data(&blockscout.connection, current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; let parameters = UpdateParameters { db: &db, @@ -159,15 +157,15 @@ mod tests { #[ignore = "needs database to run"] async fn update_total_blocks_fresh() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all("update_total_blocks_fresh").await; + let (db, blockscout) = init_db_all("update_total_blocks_fresh").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db.connection, ¤t_time) + TotalBlocks::init_recursively(&db, ¤t_time) .await .unwrap(); - fill_mock_blockscout_data(&blockscout.connection, current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; let parameters = UpdateParameters { db: &db, @@ -186,11 +184,11 @@ mod tests { #[ignore = "needs database to run"] async fn update_total_blocks_last() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all("update_total_blocks_last").await; + let (db, blockscout) = init_db_all("update_total_blocks_last").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db.connection, ¤t_time) + TotalBlocks::init_recursively(&db, ¤t_time) .await .unwrap(); @@ -200,11 +198,11 @@ mod tests { value: Set(1.to_string()), ..Default::default() }) - .exec(&db.connection as &DatabaseConnection) + .exec(&db as &DatabaseConnection) .await .unwrap(); - fill_mock_blockscout_data(&blockscout.connection, current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; let parameters = UpdateParameters { db: &db, diff --git a/stats/stats/src/charts/counters/total_contracts.rs b/stats/stats/src/charts/counters/total_contracts.rs index 841d7b6a1..36e659698 100644 --- a/stats/stats/src/charts/counters/total_contracts.rs +++ b/stats/stats/src/charts/counters/total_contracts.rs @@ -1,3 +1,4 @@ + use crate::{ data_source::{ kinds::{ @@ -28,7 +29,7 @@ impl RemoteQueryBehaviour for TotalContractsQueryBehaviour { let value = addresses::Entity::find() .filter(addresses::Column::ContractCode.is_not_null()) .filter(addresses::Column::InsertedAt.lte(cx.time)) - .count(cx.blockscout.connection.as_ref()) + .count(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)?; let timespan = cx.time.date_naive(); diff --git a/stats/stats/src/charts/counters/total_txns.rs b/stats/stats/src/charts/counters/total_txns.rs index bc8b88f01..314357077 100644 --- a/stats/stats/src/charts/counters/total_txns.rs +++ b/stats/stats/src/charts/counters/total_txns.rs @@ -10,7 +10,6 @@ use crate::{ }, range::UniversalRange, types::timespans::DateValue, - utils::MarkedDbConnection, ChartError, ChartProperties, MissingDatePolicy, Named, }; @@ -18,7 +17,8 @@ use blockscout_db::entity::{blocks, transactions}; use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{ - prelude::Expr, ColumnTrait, EntityName, EntityTrait, PaginatorTrait, QueryFilter, QuerySelect, + prelude::Expr, ColumnTrait, DatabaseConnection, EntityName, EntityTrait, PaginatorTrait, + QueryFilter, QuerySelect, }; pub struct TotalTxnsQueryBehaviour; @@ -30,7 +30,7 @@ impl RemoteQueryBehaviour for TotalTxnsQueryBehaviour { cx: &UpdateContext<'_>, _range: UniversalRange>, ) -> Result { - let blockscout = cx.blockscout.connection.as_ref(); + let blockscout = cx.blockscout; let timespan: NaiveDateTime = blocks::Entity::find() .select_only() .column_as(Expr::col(blocks::Column::Timestamp).max(), "timestamp") @@ -79,18 +79,15 @@ impl ChartProperties for Properties { pub struct TotalTxnsEstimation; impl ValueEstimation for TotalTxnsEstimation { - async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError> { + async fn estimate(blockscout: &DatabaseConnection) -> Result, ChartError> { // `now()` is more relevant when taken right before the query rather than // `cx.time` measured a bit earlier. let now = Utc::now(); - let value = query_estimated_table_rows( - blockscout.connection.as_ref(), - transactions::Entity.table_name(), - ) - .await - .map_err(ChartError::BlockscoutDB)? - .map(|n| u64::try_from(n).unwrap_or(0)) - .unwrap_or(0); + let value = query_estimated_table_rows(blockscout, transactions::Entity.table_name()) + .await + .map_err(ChartError::BlockscoutDB)? + .map(|n| u64::try_from(n).unwrap_or(0)) + .unwrap_or(0); Ok(DateValue { timespan: now.date_naive(), value: value.to_string(), diff --git a/stats/stats/src/charts/counters/yesterday_txns.rs b/stats/stats/src/charts/counters/yesterday_txns.rs index 691ee7b97..c43f892ee 100644 --- a/stats/stats/src/charts/counters/yesterday_txns.rs +++ b/stats/stats/src/charts/counters/yesterday_txns.rs @@ -1,3 +1,4 @@ + use crate::{ data_source::{ kinds::{ @@ -37,7 +38,7 @@ impl RemoteQueryBehaviour for YesterdayTxnsQuery { &cx.blockscout_applied_migrations, ); let data = Self::Output::find_by_statement(query) - .one(cx.blockscout.connection.as_ref()) + .one(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)? // no transactions for yesterday diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index ac67a2191..244dfcbc1 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -657,7 +657,7 @@ impl RemoteQueryBehaviour for QueryAllBlockTimestampRange { cx: &UpdateContext<'_>, _range: UniversalRange>, ) -> Result { - let start_timestamp = get_min_date_blockscout(cx.blockscout.connection.as_ref()) + let start_timestamp = get_min_date_blockscout(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)? .and_utc(); @@ -683,7 +683,6 @@ mod tests { simple_test::get_counter, }, types::timespans::Month, - utils::MarkedDbConnection, Named, }; use chrono::DateTime; @@ -829,8 +828,8 @@ mod tests { async fn get_counter_mock() { let _ = tracing_subscriber::fmt::try_init(); - let db = MarkedDbConnection::from_test_db(&init_db("get_counter_mock").await).unwrap(); - insert_mock_data(&db.connection).await; + let db = init_db("get_counter_mock").await; + insert_mock_data(&db).await; let current_time = dt("2022-11-12T08:08:08").and_utc(); let date = current_time.date_naive(); let cx = UpdateContext::from_params_now_or_override(UpdateParameters { diff --git a/stats/stats/src/charts/lines/native_coin_holders_growth.rs b/stats/stats/src/charts/lines/native_coin_holders_growth.rs index 9e2fc8c4e..69066949d 100644 --- a/stats/stats/src/charts/lines/native_coin_holders_growth.rs +++ b/stats/stats/src/charts/lines/native_coin_holders_growth.rs @@ -118,18 +118,16 @@ pub async fn update_sequentially_with_support_table( ) -> Result<(), ChartError> { tracing::info!(chart =% Properties::key(), "start sequential update"); let all_days = match last_accurate_point { - Some(last_row) => get_unique_ordered_days( - cx.blockscout.connection.as_ref(), - Some(last_row.timespan), - remote_fetch_timer, - ) - .await - .map_err(ChartError::BlockscoutDB)?, + Some(last_row) => { + get_unique_ordered_days(cx.blockscout, Some(last_row.timespan), remote_fetch_timer) + .await + .map_err(ChartError::BlockscoutDB)? + } None => { - clear_support_table(cx.db.connection.as_ref()) + clear_support_table(cx.db) .await .map_err(ChartError::BlockscoutDB)?; - get_unique_ordered_days(cx.blockscout.connection.as_ref(), None, remote_fetch_timer) + get_unique_ordered_days(cx.blockscout, None, remote_fetch_timer) .await .map_err(ChartError::BlockscoutDB)? } @@ -146,22 +144,14 @@ pub async fn update_sequentially_with_support_table( ); // NOTE: we update support table and chart data in one transaction // to support invariant that support table has information about last day in chart data - let db_tx = cx - .db - .connection - .begin() - .await - .map_err(ChartError::StatsDB)?; - let data: Vec = calculate_days_using_support_table( - &db_tx, - cx.blockscout.connection.as_ref(), - days.iter().copied(), - ) - .await - .map_err(|e| ChartError::Internal(e.to_string()))? - .into_iter() - .map(|result| result.active_model(chart_id, Some(min_blockscout_block))) - .collect(); + let db_tx = cx.db.begin().await.map_err(ChartError::StatsDB)?; + let data: Vec = + calculate_days_using_support_table(&db_tx, cx.blockscout, days.iter().copied()) + .await + .map_err(|e| ChartError::Internal(e.to_string()))? + .into_iter() + .map(|result| result.active_model(chart_id, Some(min_blockscout_block))) + .collect(); insert_data_many(&db_tx, data) .await .map_err(ChartError::StatsDB)?; diff --git a/stats/stats/src/charts/lines/new_accounts.rs b/stats/stats/src/charts/lines/new_accounts.rs index 24f5da7a6..6a956a97c 100644 --- a/stats/stats/src/charts/lines/new_accounts.rs +++ b/stats/stats/src/charts/lines/new_accounts.rs @@ -110,7 +110,7 @@ impl RemoteQueryBehaviour for NewAccountsQueryBehaviour { &cx.blockscout_applied_migrations, ); let mut data = DateValue::::find_by_statement(query) - .all(cx.blockscout.connection.as_ref()) + .all(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)?; // make sure that it's sorted diff --git a/stats/stats/src/charts/lines/new_block_rewards.rs b/stats/stats/src/charts/lines/new_block_rewards.rs index 225610c99..0792b7f2b 100644 --- a/stats/stats/src/charts/lines/new_block_rewards.rs +++ b/stats/stats/src/charts/lines/new_block_rewards.rs @@ -100,7 +100,7 @@ mod tests { data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, range::UniversalRange, tests::{ - init_db::init_marked_db_all, + init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data, simple_test::{map_str_tuple_to_owned, simple_test_chart}, }, @@ -137,13 +137,13 @@ mod tests { ("2023-02-01", "1"), ("2023-03-01", "1"), ]); - let (db, blockscout) = init_marked_db_all("update_new_block_rewards_monthly_int").await; + let (db, blockscout) = init_db_all("update_new_block_rewards_monthly_int").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - NewBlockRewardsMonthlyInt::init_recursively(&db.connection, ¤t_time) + NewBlockRewardsMonthlyInt::init_recursively(&db, ¤t_time) .await .unwrap(); - fill_mock_blockscout_data(&blockscout.connection, current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; let parameters = UpdateParameters { db: &db, diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index 79321a396..69eb03e4b 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -108,7 +108,7 @@ mod tests { query_dispatch::{serialize_line_points, QuerySerialized}, range::UniversalRange, tests::{ - init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data, + init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data, point_construction::dt, simple_test::simple_test_chart, }, types::ExtendedTimespanValue, @@ -117,25 +117,23 @@ mod tests { use chrono::{NaiveDate, Utc}; use entity::{chart_data, charts}; use pretty_assertions::assert_eq; - use sea_orm::{EntityTrait, Set}; + use sea_orm::{DatabaseConnection, EntityTrait, Set}; use std::str::FromStr; #[tokio::test] #[ignore = "needs database to run"] async fn update_new_blocks_recurrent() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all("update_new_blocks_recurrent").await; + let (db, blockscout) = init_db_all("update_new_blocks_recurrent").await; let current_time = chrono::DateTime::::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; - NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) + NewBlocks::init_recursively(&db, ¤t_time) .await .unwrap(); - let min_blockscout_block = get_min_block_blockscout(blockscout.connection.as_ref()) - .await - .unwrap(); + let min_blockscout_block = get_min_block_blockscout(&blockscout).await.unwrap(); // set wrong value and check, that it was rewritten chart_data::Entity::insert_many([ chart_data::ActiveModel { @@ -153,7 +151,7 @@ mod tests { ..Default::default() }, ]) - .exec(db.connection.as_ref()) + .exec(&db as &DatabaseConnection) .await .unwrap(); // set corresponding `last_updated_at` for successful partial update @@ -162,7 +160,7 @@ mod tests { last_updated_at: Set(Some(dt("2022-11-12T11:00:00").and_utc().fixed_offset())), ..Default::default() }) - .exec(db.connection.as_ref()) + .exec(&db as &DatabaseConnection) .await .unwrap(); @@ -235,12 +233,12 @@ mod tests { #[ignore = "needs database to run"] async fn update_new_blocks_fresh() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all("update_new_blocks_fresh").await; + let (db, blockscout) = init_db_all("update_new_blocks_fresh").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; - NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) + NewBlocks::init_recursively(&db, ¤t_time) .await .unwrap(); @@ -284,18 +282,16 @@ mod tests { #[ignore = "needs database to run"] async fn update_new_blocks_last() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all("update_new_blocks_last").await; + let (db, blockscout) = init_db_all("update_new_blocks_last").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; - NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) + NewBlocks::init_recursively(&db, ¤t_time) .await .unwrap(); - let min_blockscout_block = get_min_block_blockscout(blockscout.connection.as_ref()) - .await - .unwrap(); + let min_blockscout_block = get_min_block_blockscout(&blockscout).await.unwrap(); // set wrong values and check, that they weren't rewritten // except the last one chart_data::Entity::insert_many([ @@ -328,7 +324,7 @@ mod tests { ..Default::default() }, ]) - .exec(db.connection.as_ref()) + .exec(&db as &DatabaseConnection) .await .unwrap(); // set corresponding `last_updated_at` for successful partial update @@ -337,7 +333,7 @@ mod tests { last_updated_at: Set(Some(dt("2022-11-12T11:00:00").and_utc().fixed_offset())), ..Default::default() }) - .exec(db.connection.as_ref()) + .exec(&db as &DatabaseConnection) .await .unwrap(); diff --git a/stats/stats/src/charts/lines/new_txns_window.rs b/stats/stats/src/charts/lines/new_txns_window.rs index 3d9b6327b..9a61b3d99 100644 --- a/stats/stats/src/charts/lines/new_txns_window.rs +++ b/stats/stats/src/charts/lines/new_txns_window.rs @@ -61,7 +61,7 @@ impl RemoteQueryBehaviour for NewTxnsWindowQuery { let update_day = cx.time.date_naive(); let query = new_txns_window_statement(update_day, &cx.blockscout_applied_migrations); let mut data = TimespanValue::::find_by_statement(query) - .all(cx.blockscout.connection.as_ref()) + .all(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)?; // linear time for sorted sequences @@ -117,7 +117,6 @@ mod tests { point_construction::dt, simple_test::{chart_output_to_expected, map_str_tuple_to_owned, prepare_chart_test}, }, - utils::MarkedDbConnection, }; #[tokio::test] @@ -133,8 +132,8 @@ mod tests { let current_time = dt("2022-12-01T00:00:00").and_utc(); let mut parameters = UpdateParameters { - db: &MarkedDbConnection::from_test_db(&db).unwrap(), - blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), + db: &db, + blockscout: &blockscout, blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: false, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs index a75438808..f5af1d82b 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs @@ -80,8 +80,6 @@ where #[cfg(test)] mod tests { - use std::sync::Arc; - use crate::{ data_source::{types::BlockscoutMigrations, UpdateParameters}, gettable_const, @@ -89,7 +87,6 @@ mod tests { range::UniversalRange, tests::point_construction::{d_v_double, dt}, types::timespans::DateValue, - utils::MarkedDbConnection, MissingDatePolicy, Named, }; @@ -156,9 +153,7 @@ mod tests { type TestedPrevious = FilterDeducible; // db is not used in mock - let empty_db = MarkedDbConnection::in_memory(Arc::new( - sea_orm::Database::connect("sqlite::memory:").await.unwrap(), - )); + let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs index 800c348f0..d78661cf5 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs @@ -137,7 +137,7 @@ where #[cfg(test)] mod tests { - use std::{ops::Range, sync::Arc}; + use std::ops::Range; use crate::{ data_source::{ @@ -148,7 +148,6 @@ mod tests { lines::{PredefinedMockSource, PseudoRandomMockRetrieve}, tests::point_construction::{d, d_v_double, d_v_int, dt, w_v_double, week_of}, types::timespans::{DateValue, Week, WeekValue}, - utils::MarkedDbConnection, MissingDatePolicy, }; @@ -203,9 +202,7 @@ mod tests { // 8-14, 15-21, 22-28 // db is not used in mock - let db = MarkedDbConnection::in_memory(Arc::new( - sea_orm::Database::connect("sqlite::memory:").await.unwrap(), - )); + let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); let output: Vec> = TestedAverageSource::query_data( &UpdateContext::from_params_now_or_override(UpdateParameters { db: &db, @@ -253,9 +250,7 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = MarkedDbConnection::in_memory(Arc::new( - sea_orm::Database::connect("sqlite::memory:").await.unwrap(), - )); + let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, @@ -305,9 +300,7 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = MarkedDbConnection::in_memory(Arc::new( - sea_orm::Database::connect("sqlite::memory:").await.unwrap(), - )); + let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, @@ -353,9 +346,7 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = MarkedDbConnection::in_memory(Arc::new( - sea_orm::Database::connect("sqlite::memory:").await.unwrap(), - )); + let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs index 4f1549fbe..d667937ec 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs @@ -77,8 +77,6 @@ where #[cfg(test)] mod tests { - use std::sync::Arc; - use blockscout_metrics_tools::AggregateTimer; use pretty_assertions::assert_eq; @@ -89,7 +87,6 @@ mod tests { range::UniversalRange, tests::point_construction::{d_v_int, dt, w_v_int}, types::timespans::{DateValue, Week}, - utils::MarkedDbConnection, MissingDatePolicy, }; @@ -112,9 +109,7 @@ mod tests { type MockSourceWeekly = LastValueLowerResolution; // db is not used in mock - let empty_db = MarkedDbConnection::in_memory(Arc::new( - sea_orm::Database::connect("sqlite::memory:").await.unwrap(), - )); + let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs index a50fc6cfd..720042f37 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs @@ -97,8 +97,6 @@ where #[cfg(test)] mod tests { - use std::sync::Arc; - use blockscout_metrics_tools::AggregateTimer; use pretty_assertions::assert_eq; @@ -109,7 +107,6 @@ mod tests { range::UniversalRange, tests::point_construction::{d_v_int, dt, w_v_int}, types::timespans::{DateValue, Week}, - utils::MarkedDbConnection, MissingDatePolicy, }; @@ -132,9 +129,7 @@ mod tests { type MockSourceWeekly = SumLowerResolution; // db is not used in mock - let empty_db = MarkedDbConnection::in_memory(Arc::new( - sea_orm::Database::connect("sqlite::memory:").await.unwrap(), - )); + let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); let context = UpdateContext::from_params_now_or_override(UpdateParameters { db: &empty_db, diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index 93cc00694..d924e40f6 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -142,7 +142,7 @@ where cx: &UpdateContext<'_>, dependency_data_fetch_timer: &mut AggregateTimer, ) -> Result<(), ChartError> { - let metadata = get_chart_metadata(cx.db.connection.as_ref(), &ChartProps::key()).await?; + let metadata = get_chart_metadata(cx.db, &ChartProps::key()).await?; if let Some(last_updated_at) = metadata.last_updated_at { if postgres_timestamps_eq(cx.time, last_updated_at) { // no need to perform update. @@ -163,13 +163,13 @@ where } } let chart_id = metadata.id; - let min_blockscout_block = get_min_block_blockscout(cx.blockscout.connection.as_ref()) + let min_blockscout_block = get_min_block_blockscout(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)?; let last_accurate_point = last_accurate_point::( chart_id, min_blockscout_block, - cx.db.connection.as_ref(), + cx.db, cx.force_full, ChartProps::approximate_trailing_points(), ChartProps::missing_date_policy(), @@ -185,7 +185,7 @@ where ) .await?; tracing::info!(chart =% ChartProps::key(), "updating chart metadata"); - Update::update_metadata(cx.db.connection.as_ref(), chart_id, cx.time).await?; + Update::update_metadata(cx.db, chart_id, cx.time).await?; Ok(()) } @@ -326,7 +326,7 @@ mod tests { DataSource, UpdateContext, UpdateParameters, }, gettable_const, - tests::{init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data}, + tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, types::{timespans::DateValue, TimespanValue}, update_group::{SyncUpdateGroup, UpdateGroup}, ChartError, ChartProperties, Named, @@ -376,7 +376,7 @@ mod tests { value: "0".to_owned(), }; let value = data.active_model(chart_id, Some(min_blockscout_block)); - insert_data_many(cx.db.connection.as_ref(), vec![value]) + insert_data_many(cx.db, vec![value]) .await .map_err(ChartError::StatsDB)?; Ok(()) @@ -435,11 +435,10 @@ mod tests { #[ignore = "needs database to run"] async fn update_itself_is_triggered_once_per_group() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = - init_marked_db_all("update_itself_is_triggered_once_per_group").await; + let (db, blockscout) = init_db_all("update_itself_is_triggered_once_per_group").await; let current_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; let enabled = HashSet::from( [TestedChartProps::key(), ChartDependedOnTestedProps::key()].map(|l| l.to_owned()), ); @@ -450,7 +449,7 @@ mod tests { .collect(); let group = SyncUpdateGroup::new(&mutexes, Arc::new(TestUpdateGroup)).unwrap(); group - .create_charts_with_mutexes(db.connection.as_ref(), Some(current_time), &enabled) + .create_charts_with_mutexes(&db, Some(current_time), &enabled) .await .unwrap(); diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index b39ed7a4b..44269570a 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -1,13 +1,13 @@ use std::{fmt::Debug, marker::PhantomData}; use chrono::{DateTime, Utc}; +use sea_orm::DatabaseConnection; use crate::{ charts::db_interaction::read::{get_counter_data, get_line_chart_data}, data_source::{kinds::local_db::parameter_traits::QueryBehaviour, UpdateContext}, range::UniversalRange, types::{timespans::DateValue, ExtendedTimespanValue, Timespan}, - utils::MarkedDbConnection, ChartError, ChartProperties, RequestedPointsLimit, }; @@ -48,7 +48,7 @@ where let start = start.map(|s| C::Resolution::from_date(s.date_naive())); let end = end.map(|e| C::Resolution::from_date(e.date_naive())); let values = get_line_chart_data::( - cx.db.connection.as_ref(), + cx.db, &C::name(), start, end, @@ -75,7 +75,7 @@ impl QueryBehaviour for DefaultQueryLast { _fill_missing_dates: bool, ) -> Result { let value = get_counter_data( - cx.db.connection.as_ref(), + cx.db, &C::name(), Some(cx.time.date_naive()), C::missing_date_policy(), @@ -91,7 +91,7 @@ impl QueryBehaviour for DefaultQueryLast { #[trait_variant::make(Send)] pub trait ValueEstimation { - async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError>; + async fn estimate(blockscout: &DatabaseConnection) -> Result, ChartError>; } pub struct QueryLastWithEstimationFallback(PhantomData<(E, C)>) @@ -113,7 +113,7 @@ where _fill_missing_dates: bool, ) -> Result { let value = match get_counter_data( - cx.db.connection.as_ref(), + cx.db, &C::name(), Some(cx.time.date_naive()), C::missing_date_policy(), @@ -134,12 +134,13 @@ mod tests { use chrono::NaiveDate; use entity::sea_orm_active_enums::ChartType; use pretty_assertions::assert_eq; + use sea_orm::DatabaseConnection; use super::*; use crate::{ data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, - tests::init_db::init_marked_db_all, + tests::init_db::init_db_all, types::timespans::DateValue, ChartError, MissingDatePolicy, Named, }; @@ -148,7 +149,7 @@ mod tests { #[ignore = "needs database to run"] async fn fallback_query_works() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all("fallback_query_works").await; + let (db, blockscout) = init_db_all("fallback_query_works").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let parameters = UpdateParameters { @@ -171,7 +172,7 @@ mod tests { impl ValueEstimation for TestFallback { async fn estimate( - _blockscout: &MarkedDbConnection, + _blockscout: &DatabaseConnection, ) -> Result, ChartError> { Ok(expected_estimate()) } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs index d4e83821d..78eaaa4e1 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs @@ -69,7 +69,7 @@ where let update_range_start = match update_from { Some(d) => d, None => ChartProps::Resolution::from_date( - get_min_date_blockscout(cx.blockscout.connection.as_ref()) + get_min_date_blockscout(cx.blockscout) .await .map(|time| time.date()) .map_err(ChartError::BlockscoutDB)?, @@ -107,12 +107,7 @@ where ) .await?; // for query in `get_previous_step_last_point` to work correctly - Self::update_metadata( - cx.db.connection.as_ref(), - chart_id, - range.into_date_time_range().end, - ) - .await?; + Self::update_metadata(cx.db, chart_id, range.into_date_time_range().end).await?; let elapsed: std::time::Duration = now.elapsed(); tracing::info!( found =? found, @@ -184,7 +179,7 @@ where let resolution_data = ResolutionDep::query_data(cx, query_range, dependency_data_fetch_timer).await?; let found = BatchStep::batch_update_values_step_with( - cx.db.connection.as_ref(), + cx.db, chart_id, cx.time, min_blockscout_block, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs index f3b3c4a16..98c427628 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs @@ -28,7 +28,7 @@ where // range doesn't make sense there; thus is not used let data = MainDep::query_data(cx, UniversalRange::full(), remote_fetch_timer).await?; let value = data.active_model(chart_id, Some(min_blockscout_block)); - insert_data_many(cx.db.connection.as_ref(), vec![value]) + insert_data_many(cx.db, vec![value]) .await .map_err(ChartError::StatsDB)?; Ok(()) diff --git a/stats/stats/src/data_source/kinds/remote_db/query/all.rs b/stats/stats/src/data_source/kinds/remote_db/query/all.rs index 0583240e4..a16cdc491 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/all.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/all.rs @@ -60,7 +60,7 @@ where data_source_query_range_to_db_statement_range::(cx, range).await?; let query = S::get_statement(query_range, &cx.blockscout_applied_migrations); let mut data = TimespanValue::::find_by_statement(query) - .all(cx.blockscout.connection.as_ref()) + .all(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)?; // linear time for sorted sequences diff --git a/stats/stats/src/data_source/kinds/remote_db/query/each.rs b/stats/stats/src/data_source/kinds/remote_db/query/each.rs index ea8dcf397..509e50c72 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/each.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/each.rs @@ -63,7 +63,7 @@ where for point_range in points { let query = S::get_statement(point_range.clone(), &cx.blockscout_applied_migrations); let point_value = ValueWrapper::::find_by_statement(query) - .one(cx.blockscout.connection.as_ref()) + .one(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)?; if let Some(ValueWrapper { value }) = point_value { diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index 9be5b0798..32bc05e22 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -47,7 +47,7 @@ where ) -> Result, ChartError> { let query = S::get_statement(&cx.blockscout_applied_migrations); let data = TimespanValue::::find_by_statement(query) - .one(cx.blockscout.connection.as_ref()) + .one(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)? .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; @@ -87,7 +87,7 @@ where } else { let find_by_statement = Value::find_by_statement(query.clone()); let value = find_by_statement - .one(cx.blockscout.connection.as_ref()) + .one(cx.blockscout) .await .map_err(ChartError::BlockscoutDB)? .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; @@ -104,7 +104,7 @@ where #[cfg(test)] mod test { - use std::{collections::BTreeMap, ops::Range, sync::Arc}; + use std::{collections::BTreeMap, ops::Range}; use chrono::{DateTime, Utc}; use pretty_assertions::assert_eq; @@ -119,7 +119,6 @@ mod test { range::UniversalRange, tests::point_construction::dt, types::TimespanValue, - utils::MarkedDbConnection, }; use super::PullOne24hCached; @@ -141,7 +140,7 @@ mod test { let expected = WrappedValue { value: "value1".to_string(), }; - let mock_db = MockDatabase::new(DatabaseBackend::Postgres) + let db = MockDatabase::new(DatabaseBackend::Postgres) .append_query_results([ // First query result vec![BTreeMap::from([( @@ -152,8 +151,6 @@ mod test { vec![BTreeMap::from([("value", sea_orm::Value::from("value2"))])], ]) .into_connection(); - let db = MarkedDbConnection::main_connection(Arc::new(mock_db)); - let time = dt("2023-01-01T00:00:00").and_utc(); let cx = UpdateContext::from_params_now_or_override(UpdateParameters { db: &db, diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index 35284bbb5..4f1b4b620 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -34,7 +34,7 @@ use crate::{ types::BlockscoutMigrations, }, define_and_impl_resolution_properties, - tests::{init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data}, + tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, types::timespans::{DateValue, Month, Week, Year}, update_group::{SyncUpdateGroup, UpdateGroup}, utils::{produce_filter_and_values, sql_with_range_filter_opt}, @@ -270,10 +270,10 @@ construct_update_group!(ExampleUpdateGroup { #[ignore = "needs database to run"] async fn update_examples() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all("update_examples").await; + let (db, blockscout) = init_db_all("update_examples").await; let current_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; let enabled = HashSet::from( [ NewContractsChartProperties::key(), @@ -293,7 +293,7 @@ async fn update_examples() { .collect(); let group = SyncUpdateGroup::new(&mutexes, Arc::new(ExampleUpdateGroup)).unwrap(); group - .create_charts_with_mutexes(db.connection.as_ref(), None, &enabled) + .create_charts_with_mutexes(&db, None, &enabled) .await .unwrap(); diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 0ec5c6f1c..a43ede7fc 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -8,12 +8,12 @@ use sea_orm::{ use tokio::sync::Mutex; use tracing::warn; -use crate::{counters::TxnsStatsValue, utils::MarkedDbConnection}; +use crate::counters::TxnsStatsValue; #[derive(Clone)] pub struct UpdateParameters<'a> { - pub db: &'a MarkedDbConnection, - pub blockscout: &'a MarkedDbConnection, + pub db: &'a DatabaseConnection, + pub blockscout: &'a DatabaseConnection, pub blockscout_applied_migrations: BlockscoutMigrations, /// If `None`, it will be measured at the start of update /// (i.e. after taking mutexes) @@ -24,8 +24,8 @@ pub struct UpdateParameters<'a> { #[derive(Clone)] pub struct UpdateContext<'a> { - pub db: &'a MarkedDbConnection, - pub blockscout: &'a MarkedDbConnection, + pub db: &'a DatabaseConnection, + pub blockscout: &'a DatabaseConnection, pub blockscout_applied_migrations: BlockscoutMigrations, pub cache: UpdateCache, /// Update time diff --git a/stats/stats/src/tests/init_db.rs b/stats/stats/src/tests/init_db.rs index 29473a678..655511e45 100644 --- a/stats/stats/src/tests/init_db.rs +++ b/stats/stats/src/tests/init_db.rs @@ -1,7 +1,5 @@ use blockscout_service_launcher::test_database::TestDbGuard; -use crate::utils::MarkedDbConnection; - pub async fn init_db_all(name: &str) -> (TestDbGuard, TestDbGuard) { let db = init_db(name).await; let blockscout = @@ -13,11 +11,3 @@ pub async fn init_db_all(name: &str) -> (TestDbGuard, TestDbGuard) { pub async fn init_db(name: &str) -> TestDbGuard { TestDbGuard::new::(name).await } - -pub async fn init_marked_db_all(name: &str) -> (MarkedDbConnection, MarkedDbConnection) { - let (db, blockscout) = init_db_all(name).await; - ( - MarkedDbConnection::from_test_db(&db).unwrap(), - MarkedDbConnection::from_test_db(&blockscout).unwrap(), - ) -} diff --git a/stats/stats/src/tests/simple_test.rs b/stats/stats/src/tests/simple_test.rs index 6edbebfee..80896fe2b 100644 --- a/stats/stats/src/tests/simple_test.rs +++ b/stats/stats/src/tests/simple_test.rs @@ -1,7 +1,4 @@ -use super::{ - init_db::{init_db_all, init_marked_db_all}, - mock_blockscout::fill_mock_blockscout_data, -}; +use super::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}; use crate::{ data_source::{ source::DataSource, @@ -10,13 +7,12 @@ use crate::{ query_dispatch::QuerySerialized, range::UniversalRange, types::{timespans::DateValue, Timespan}, - utils::MarkedDbConnection, ChartProperties, }; use blockscout_service_launcher::test_database::TestDbGuard; use chrono::{DateTime, NaiveDateTime, Utc}; use pretty_assertions::assert_eq; -use sea_orm::{ConnectionTrait, DbBackend, Statement}; +use sea_orm::{ConnectionTrait, DatabaseConnection, DbBackend, Statement}; use stats_proto::blockscout::stats::v1::Point; use std::{fmt::Debug, str::FromStr}; @@ -84,8 +80,8 @@ where fill_mock_blockscout_data(&blockscout, current_date).await; let mut parameters = UpdateParameters { - db: &MarkedDbConnection::from_test_db(&db).unwrap(), - blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), + db: &db, + blockscout: &blockscout, blockscout_applied_migrations: migrations, update_time_override: Some(current_time), force_full: true, @@ -119,8 +115,8 @@ where /// /// Tests that force update with existing data works correctly pub async fn dirty_force_update_and_check( - db: &TestDbGuard, - blockscout: &TestDbGuard, + db: &DatabaseConnection, + blockscout: &DatabaseConnection, expected: Vec<(&str, &str)>, update_time_override: Option>, ) where @@ -134,8 +130,8 @@ pub async fn dirty_force_update_and_check( update_time_override.unwrap_or(DateTime::from_str("2023-03-01T12:00:01Z").unwrap()); let parameters = UpdateParameters { - db: &MarkedDbConnection::from_test_db(db).unwrap(), - blockscout: &MarkedDbConnection::from_test_db(blockscout).unwrap(), + db: &db, + blockscout: &blockscout, blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: true, @@ -224,15 +220,13 @@ async fn ranged_test_chart_inner( { let _ = tracing_subscriber::fmt::try_init(); let expected = map_str_tuple_to_owned(expected); - let (db, blockscout) = init_marked_db_all(test_name).await; + let (db, blockscout) = init_db_all(test_name).await; let max_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_time = update_time.map(|t| t.and_utc()).unwrap_or(max_time); let max_date = max_time.date_naive(); let range = { from.into_time_range().start..to.into_time_range().end }; - C::init_recursively(db.connection.as_ref(), ¤t_time) - .await - .unwrap(); - fill_mock_blockscout_data(blockscout.connection.as_ref(), max_date).await; + C::init_recursively(&db, ¤t_time).await.unwrap(); + fill_mock_blockscout_data(&blockscout, max_date).await; let mut parameters = UpdateParameters { db: &db, @@ -316,8 +310,8 @@ async fn simple_test_counter_inner( fill_mock_blockscout_data(&blockscout, max_date).await; let mut parameters = UpdateParameters { - db: &MarkedDbConnection::from_test_db(&db).unwrap(), - blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), + db: &db, + blockscout: &blockscout, blockscout_applied_migrations: migrations, update_time_override: Some(current_time), force_full: true, @@ -339,20 +333,17 @@ where C: DataSource + ChartProperties + QuerySerialized>, { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_marked_db_all(test_name).await; + let (db, blockscout) = init_db_all(test_name).await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - C::init_recursively(&db.connection, ¤t_time) - .await - .unwrap(); + C::init_recursively(&db, ¤t_time).await.unwrap(); - fill_mock_blockscout_data(&blockscout.connection, current_date).await; + fill_mock_blockscout_data(&blockscout, current_date).await; // need to analyze or vacuum for `reltuples` to be updated. // source: https://www.postgresql.org/docs/9.3/planner-stats.html let _ = blockscout - .connection .execute(Statement::from_string(DbBackend::Postgres, "ANALYZE;")) .await .unwrap(); diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index 203bb61ff..a79dec1ae 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, NaiveDate, NaiveTime, Utc}; use sea_orm::Value; -use std::{ops::Range, sync::Arc}; +use std::ops::Range; // this const is not public in `chrono` for some reason pub const NANOS_PER_SEC: i32 = 1_000_000_000; @@ -12,45 +12,6 @@ pub fn day_start(date: &NaiveDate) -> DateTime { .and_utc() } -// todo: remove marked part if not used until May 2025. -// probably rename to some wrapper of db connection to add some other -// stuff if necessary (or use UpdateContext as a place to extend the context) -/// Database connection with a mark of what database it is. -/// -/// Used to separate caching for different databases to -/// prevent data clashes when running unit tests concurrently. -#[derive(Debug, Clone)] -pub struct MarkedDbConnection { - pub connection: Arc, - pub db_name: String, -} - -impl MarkedDbConnection { - #[cfg(any(feature = "test-utils", test))] - pub fn from_test_db( - guard: &blockscout_service_launcher::test_database::TestDbGuard, - ) -> Option { - Some(Self { - connection: guard.client(), - db_name: guard.db_url().split("/").last()?.to_owned(), - }) - } - - pub fn main_connection(inner: Arc) -> Self { - Self { - connection: inner, - db_name: "main".to_owned(), - } - } - - pub fn in_memory(inner: Arc) -> Self { - Self { - connection: inner, - db_name: "in_memory".to_owned(), - } - } -} - /// Used inside [`sql_with_range_filter_opt`] /// /// `filter_arg_number_start = len(arg)+1 // (length of other args + 1)` From e9859399648f97adb50460cadf400a7699004836 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 20 Dec 2024 01:09:01 +0900 Subject: [PATCH 14/17] fmt --- stats/stats/src/charts/counters/total_contracts.rs | 1 - stats/stats/src/charts/counters/txns_fee_24h.rs | 7 +++---- stats/stats/src/charts/counters/yesterday_txns.rs | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/stats/stats/src/charts/counters/total_contracts.rs b/stats/stats/src/charts/counters/total_contracts.rs index 36e659698..2f856311b 100644 --- a/stats/stats/src/charts/counters/total_contracts.rs +++ b/stats/stats/src/charts/counters/total_contracts.rs @@ -1,4 +1,3 @@ - use crate::{ data_source::{ kinds::{ diff --git a/stats/stats/src/charts/counters/txns_fee_24h.rs b/stats/stats/src/charts/counters/txns_fee_24h.rs index c09bc0a12..297ec4480 100644 --- a/stats/stats/src/charts/counters/txns_fee_24h.rs +++ b/stats/stats/src/charts/counters/txns_fee_24h.rs @@ -1,9 +1,8 @@ - use crate::{ data_source::kinds::{ - data_manipulation::map::{Map, MapFunction, MapToString, UnwrapOr}, - local_db::DirectPointLocalDbChartSource, - }, + data_manipulation::map::{Map, MapFunction, MapToString, UnwrapOr}, + local_db::DirectPointLocalDbChartSource, + }, gettable_const, types::TimespanValue, ChartError, ChartProperties, MissingDatePolicy, Named, diff --git a/stats/stats/src/charts/counters/yesterday_txns.rs b/stats/stats/src/charts/counters/yesterday_txns.rs index c43f892ee..e425aca5e 100644 --- a/stats/stats/src/charts/counters/yesterday_txns.rs +++ b/stats/stats/src/charts/counters/yesterday_txns.rs @@ -1,4 +1,3 @@ - use crate::{ data_source::{ kinds::{ From 77266de8cea27ba3742f3679f2f6b72ba539fcb2 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 20 Dec 2024 01:12:04 +0900 Subject: [PATCH 15/17] clippy --- stats/stats/src/data_source/types.rs | 2 +- stats/stats/src/tests/simple_test.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index a43ede7fc..c11444507 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -202,7 +202,7 @@ macro_rules! impl_cacheable_wrapped { impl_cacheable_wrapped!(WrappedValue, ValueString); impl_cacheable_wrapped!(WrappedValue>, ValueOptionF64); -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct UpdateCache { inner: Arc>>, } diff --git a/stats/stats/src/tests/simple_test.rs b/stats/stats/src/tests/simple_test.rs index 80896fe2b..92a48267e 100644 --- a/stats/stats/src/tests/simple_test.rs +++ b/stats/stats/src/tests/simple_test.rs @@ -130,8 +130,8 @@ pub async fn dirty_force_update_and_check( update_time_override.unwrap_or(DateTime::from_str("2023-03-01T12:00:01Z").unwrap()); let parameters = UpdateParameters { - db: &db, - blockscout: &blockscout, + db, + blockscout, blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: true, From 6228ed9655122a45c8b7ebdf95e1797ccffb5599 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 20 Dec 2024 22:14:58 +0900 Subject: [PATCH 16/17] comment --- stats/stats/src/data_source/types.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index c11444507..9aaac0aa1 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -202,6 +202,11 @@ macro_rules! impl_cacheable_wrapped { impl_cacheable_wrapped!(WrappedValue, ValueString); impl_cacheable_wrapped!(WrappedValue>, ValueOptionF64); +/// There is no cache invalidation logic, because the cache is +/// expected to be constructed from scratch on each group update +/// and dropped after the update. +/// +/// Also see a [`crate::construct_update_group!`] implementation #[derive(Clone, Debug, Default)] pub struct UpdateCache { inner: Arc>>, From 82ca829fbd7de9309d01d1da607bf11c0964b4d2 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov Date: Fri, 20 Dec 2024 22:20:09 +0900 Subject: [PATCH 17/17] refactor 24 hours stats --- .../charts/counters/average_txn_fee_24h.rs | 166 ------------------ stats/stats/src/charts/counters/mod.rs | 12 +- .../txns_stats_24h/average_txn_fee_24h.rs | 95 ++++++++++ .../src/charts/counters/txns_stats_24h/mod.rs | 78 ++++++++ .../{ => txns_stats_24h}/new_txns_24h.rs | 2 +- .../{ => txns_stats_24h}/txns_fee_24h.rs | 2 +- 6 files changed, 180 insertions(+), 175 deletions(-) delete mode 100644 stats/stats/src/charts/counters/average_txn_fee_24h.rs create mode 100644 stats/stats/src/charts/counters/txns_stats_24h/average_txn_fee_24h.rs create mode 100644 stats/stats/src/charts/counters/txns_stats_24h/mod.rs rename stats/stats/src/charts/counters/{ => txns_stats_24h}/new_txns_24h.rs (97%) rename stats/stats/src/charts/counters/{ => txns_stats_24h}/txns_fee_24h.rs (97%) diff --git a/stats/stats/src/charts/counters/average_txn_fee_24h.rs b/stats/stats/src/charts/counters/average_txn_fee_24h.rs deleted file mode 100644 index e371109d1..000000000 --- a/stats/stats/src/charts/counters/average_txn_fee_24h.rs +++ /dev/null @@ -1,166 +0,0 @@ -use std::ops::Range; - -use crate::{ - data_source::{ - kinds::{ - data_manipulation::map::{Map, MapFunction, MapToString, UnwrapOr}, - local_db::DirectPointLocalDbChartSource, - remote_db::{PullOne24hCached, RemoteDatabaseSource, StatementFromRange}, - }, - types::BlockscoutMigrations, - }, - gettable_const, - types::TimespanValue, - ChartProperties, MissingDatePolicy, Named, -}; -use blockscout_db::entity::{blocks, transactions}; -use chrono::{DateTime, NaiveDate, Utc}; -use entity::sea_orm_active_enums::ChartType; -use migration::{Alias, Func}; -use sea_orm::{DbBackend, FromQueryResult, IntoSimpleExpr, QuerySelect, QueryTrait, Statement}; - -const ETHER: i64 = i64::pow(10, 18); - -pub struct TxnsStatsStatement; - -impl StatementFromRange for TxnsStatsStatement { - fn get_statement( - range: Option>>, - completed_migrations: &BlockscoutMigrations, - ) -> Statement { - use sea_orm::prelude::*; - - let fee_query = Expr::cust_with_exprs( - "COALESCE($1, $2 + LEAST($3, $4))", - [ - transactions::Column::GasPrice.into_simple_expr(), - blocks::Column::BaseFeePerGas.into_simple_expr(), - transactions::Column::MaxPriorityFeePerGas.into_simple_expr(), - transactions::Column::MaxFeePerGas - .into_simple_expr() - .sub(blocks::Column::BaseFeePerGas.into_simple_expr()), - ], - ) - .mul(transactions::Column::GasUsed.into_simple_expr()); - - let count_query = Func::count(transactions::Column::Hash.into_simple_expr()); - let sum_query = Expr::expr(Func::sum(fee_query.clone())) - .div(ETHER) - .cast_as(Alias::new("FLOAT")); - let avg_query = Expr::expr(Func::avg(fee_query)) - .div(ETHER) - .cast_as(Alias::new("FLOAT")); - - let base_query = blocks::Entity::find().inner_join(transactions::Entity); - let mut query = base_query - .select_only() - .expr_as(count_query, "count") - .expr_as(sum_query, "fee_sum") - .expr_as(avg_query, "fee_average") - .to_owned(); - - if let Some(r) = range { - if completed_migrations.denormalization { - query = query - .filter(transactions::Column::BlockTimestamp.lt(r.end)) - .filter(transactions::Column::BlockTimestamp.gte(r.start)) - } - query = query - .filter(blocks::Column::Timestamp.lt(r.end)) - .filter(blocks::Column::Timestamp.gte(r.start)); - } - - query.build(DbBackend::Postgres) - } -} - -#[derive(Debug, Clone, FromQueryResult)] -pub struct TxnsStatsValue { - pub count: i64, - pub fee_average: Option, - pub fee_sum: Option, -} - -pub type Txns24hStats = RemoteDatabaseSource>; - -pub struct ExtractAverage; - -impl MapFunction> for ExtractAverage { - type Output = TimespanValue>; - - fn function( - inner_data: TimespanValue, - ) -> Result { - Ok(TimespanValue { - timespan: inner_data.timespan, - value: inner_data.value.fee_average, - }) - } -} - -pub type AverageTxnFee24hExtracted = Map; - -pub struct Properties; - -impl Named for Properties { - fn name() -> String { - "averageTxnFee24h".into() - } -} - -impl ChartProperties for Properties { - type Resolution = NaiveDate; - - fn chart_type() -> ChartType { - ChartType::Counter - } - fn missing_date_policy() -> MissingDatePolicy { - MissingDatePolicy::FillZero - } -} - -gettable_const!(Zero: f64 = 0.0); - -pub type AverageTxnFee24h = DirectPointLocalDbChartSource< - MapToString>, - Properties, ->; - -#[cfg(test)] -mod tests { - use super::*; - use crate::tests::{point_construction::dt, simple_test::simple_test_counter}; - - #[tokio::test] - #[ignore = "needs database to run"] - async fn update_average_txns_fee_1() { - simple_test_counter::( - "update_average_txns_fee_1", - "0.000023592592569", - None, - ) - .await; - } - - #[tokio::test] - #[ignore = "needs database to run"] - async fn update_average_txns_fee_2() { - simple_test_counter::( - "update_average_txns_fee_2", - "0.00006938997814411765", - Some(dt("2022-11-11T16:00:00")), - ) - .await; - } - - #[tokio::test] - #[ignore = "needs database to run"] - async fn update_average_txns_fee_3() { - simple_test_counter::( - "update_average_txns_fee_3", - "0", - Some(dt("2024-10-10T00:00:00")), - ) - .await; - } -} diff --git a/stats/stats/src/charts/counters/mod.rs b/stats/stats/src/charts/counters/mod.rs index f66b23bb1..3b4d450fe 100644 --- a/stats/stats/src/charts/counters/mod.rs +++ b/stats/stats/src/charts/counters/mod.rs @@ -1,9 +1,7 @@ mod average_block_time; -mod average_txn_fee_24h; mod completed_txns; mod last_new_contracts; mod last_new_verified_contracts; -mod new_txns_24h; mod pending_txns; mod total_accounts; mod total_addresses; @@ -15,19 +13,16 @@ mod total_operational_txns; mod total_tokens; mod total_txns; mod total_verified_contracts; -mod txns_fee_24h; +mod txns_stats_24h; mod yesterday_txns; #[cfg(test)] mod mock; pub use average_block_time::AverageBlockTime; -pub use average_txn_fee_24h::AverageTxnFee24h; -pub(crate) use average_txn_fee_24h::TxnsStatsValue; pub use completed_txns::CompletedTxns; pub use last_new_contracts::LastNewContracts; pub use last_new_verified_contracts::LastNewVerifiedContracts; -pub use new_txns_24h::NewTxns24h; pub use pending_txns::PendingTxns; pub use total_accounts::TotalAccounts; pub use total_addresses::TotalAddresses; @@ -39,7 +34,10 @@ pub use total_operational_txns::TotalOperationalTxns; pub use total_tokens::TotalTokens; pub use total_txns::{TotalTxns, TotalTxnsInt}; pub use total_verified_contracts::TotalVerifiedContracts; -pub use txns_fee_24h::TxnsFee24h; +pub(crate) use txns_stats_24h::TxnsStatsValue; +pub use txns_stats_24h::{ + average_txn_fee_24h::AverageTxnFee24h, new_txns_24h::NewTxns24h, txns_fee_24h::TxnsFee24h, +}; pub use yesterday_txns::YesterdayTxns; #[cfg(test)] diff --git a/stats/stats/src/charts/counters/txns_stats_24h/average_txn_fee_24h.rs b/stats/stats/src/charts/counters/txns_stats_24h/average_txn_fee_24h.rs new file mode 100644 index 000000000..34c66f1d4 --- /dev/null +++ b/stats/stats/src/charts/counters/txns_stats_24h/average_txn_fee_24h.rs @@ -0,0 +1,95 @@ +use crate::{ + data_source::kinds::{ + data_manipulation::map::{Map, MapFunction, MapToString, UnwrapOr}, + local_db::DirectPointLocalDbChartSource, + }, + gettable_const, + types::TimespanValue, + ChartProperties, MissingDatePolicy, Named, +}; +use chrono::NaiveDate; +use entity::sea_orm_active_enums::ChartType; + +use super::{Txns24hStats, TxnsStatsValue}; + +pub struct ExtractAverage; + +impl MapFunction> for ExtractAverage { + type Output = TimespanValue>; + + fn function( + inner_data: TimespanValue, + ) -> Result { + Ok(TimespanValue { + timespan: inner_data.timespan, + value: inner_data.value.fee_average, + }) + } +} + +pub type AverageTxnFee24hExtracted = Map; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "averageTxnFee24h".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillZero + } +} + +gettable_const!(Zero: f64 = 0.0); + +pub type AverageTxnFee24h = DirectPointLocalDbChartSource< + MapToString>, + Properties, +>; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{point_construction::dt, simple_test::simple_test_counter}; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_average_txns_fee_1() { + simple_test_counter::( + "update_average_txns_fee_1", + "0.000023592592569", + None, + ) + .await; + } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_average_txns_fee_2() { + simple_test_counter::( + "update_average_txns_fee_2", + "0.00006938997814411765", + Some(dt("2022-11-11T16:00:00")), + ) + .await; + } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_average_txns_fee_3() { + simple_test_counter::( + "update_average_txns_fee_3", + "0", + Some(dt("2024-10-10T00:00:00")), + ) + .await; + } +} diff --git a/stats/stats/src/charts/counters/txns_stats_24h/mod.rs b/stats/stats/src/charts/counters/txns_stats_24h/mod.rs new file mode 100644 index 000000000..19bf1d6a3 --- /dev/null +++ b/stats/stats/src/charts/counters/txns_stats_24h/mod.rs @@ -0,0 +1,78 @@ +use std::ops::Range; + +use crate::data_source::{ + kinds::remote_db::{PullOne24hCached, RemoteDatabaseSource, StatementFromRange}, + types::BlockscoutMigrations, +}; +use blockscout_db::entity::{blocks, transactions}; +use chrono::{DateTime, Utc}; +use migration::{Alias, Func}; +use sea_orm::{DbBackend, FromQueryResult, IntoSimpleExpr, QuerySelect, QueryTrait, Statement}; + +pub mod average_txn_fee_24h; +pub mod new_txns_24h; +pub mod txns_fee_24h; + +const ETHER: i64 = i64::pow(10, 18); + +pub struct TxnsStatsStatement; + +impl StatementFromRange for TxnsStatsStatement { + fn get_statement( + range: Option>>, + completed_migrations: &BlockscoutMigrations, + ) -> Statement { + use sea_orm::prelude::*; + + let fee_query = Expr::cust_with_exprs( + "COALESCE($1, $2 + LEAST($3, $4))", + [ + transactions::Column::GasPrice.into_simple_expr(), + blocks::Column::BaseFeePerGas.into_simple_expr(), + transactions::Column::MaxPriorityFeePerGas.into_simple_expr(), + transactions::Column::MaxFeePerGas + .into_simple_expr() + .sub(blocks::Column::BaseFeePerGas.into_simple_expr()), + ], + ) + .mul(transactions::Column::GasUsed.into_simple_expr()); + + let count_query = Func::count(transactions::Column::Hash.into_simple_expr()); + let sum_query = Expr::expr(Func::sum(fee_query.clone())) + .div(ETHER) + .cast_as(Alias::new("FLOAT")); + let avg_query = Expr::expr(Func::avg(fee_query)) + .div(ETHER) + .cast_as(Alias::new("FLOAT")); + + let base_query = blocks::Entity::find().inner_join(transactions::Entity); + let mut query = base_query + .select_only() + .expr_as(count_query, "count") + .expr_as(sum_query, "fee_sum") + .expr_as(avg_query, "fee_average") + .to_owned(); + + if let Some(r) = range { + if completed_migrations.denormalization { + query = query + .filter(transactions::Column::BlockTimestamp.lt(r.end)) + .filter(transactions::Column::BlockTimestamp.gte(r.start)) + } + query = query + .filter(blocks::Column::Timestamp.lt(r.end)) + .filter(blocks::Column::Timestamp.gte(r.start)); + } + + query.build(DbBackend::Postgres) + } +} + +#[derive(Debug, Clone, FromQueryResult)] +pub struct TxnsStatsValue { + pub count: i64, + pub fee_average: Option, + pub fee_sum: Option, +} + +pub type Txns24hStats = RemoteDatabaseSource>; diff --git a/stats/stats/src/charts/counters/new_txns_24h.rs b/stats/stats/src/charts/counters/txns_stats_24h/new_txns_24h.rs similarity index 97% rename from stats/stats/src/charts/counters/new_txns_24h.rs rename to stats/stats/src/charts/counters/txns_stats_24h/new_txns_24h.rs index ad8397a42..3735eb76b 100644 --- a/stats/stats/src/charts/counters/new_txns_24h.rs +++ b/stats/stats/src/charts/counters/txns_stats_24h/new_txns_24h.rs @@ -9,7 +9,7 @@ use crate::{ use chrono::NaiveDate; use entity::sea_orm_active_enums::ChartType; -use super::{average_txn_fee_24h::Txns24hStats, TxnsStatsValue}; +use super::{Txns24hStats, TxnsStatsValue}; pub struct ExtractCount; diff --git a/stats/stats/src/charts/counters/txns_fee_24h.rs b/stats/stats/src/charts/counters/txns_stats_24h/txns_fee_24h.rs similarity index 97% rename from stats/stats/src/charts/counters/txns_fee_24h.rs rename to stats/stats/src/charts/counters/txns_stats_24h/txns_fee_24h.rs index 297ec4480..b98d3912a 100644 --- a/stats/stats/src/charts/counters/txns_fee_24h.rs +++ b/stats/stats/src/charts/counters/txns_stats_24h/txns_fee_24h.rs @@ -10,7 +10,7 @@ use crate::{ use chrono::NaiveDate; use entity::sea_orm_active_enums::ChartType; -use super::{average_txn_fee_24h::Txns24hStats, TxnsStatsValue}; +use super::{Txns24hStats, TxnsStatsValue}; pub struct ExtractSum;