diff --git a/stats/config/charts.json b/stats/config/charts.json index 6ffcac3ba..056b24224 100644 --- a/stats/config/charts.json +++ b/stats/config/charts.json @@ -111,6 +111,10 @@ "title": "New transactions", "description": "Number of new transactions" }, + "new_txns_window": { + "title": "Daily transactions", + "description": "Number of transactions yesterday (0:00 - 23:59 UTC). The chart displays daily transactions for the past 30 days." + }, "txns_growth": { "title": "Number of transactions", "description": "Cumulative transaction growth over time" diff --git a/stats/config/layout.json b/stats/config/layout.json index 0ce20291f..56747311f 100644 --- a/stats/config/layout.json +++ b/stats/config/layout.json @@ -39,7 +39,8 @@ "txns_growth", "new_operational_txns", "operational_txns_growth", - "txns_success_rate" + "txns_success_rate", + "new_txns_window" ] }, { diff --git a/stats/config/update_groups.json b/stats/config/update_groups.json index 18e7b997e..c823856a1 100644 --- a/stats/config/update_groups.json +++ b/stats/config/update_groups.json @@ -9,6 +9,7 @@ "total_txns_group": "0 5 */2 * * * *", "total_operational_txns_group": "0 5 1 * * * *", "yesterday_txns_group": "0 8 0 * * * *", + "new_txns_window_group": "0 8 0 * * * *", "active_recurring_accounts_daily_recurrence_60_days_group": "0 0 2 * * * *", "active_recurring_accounts_daily_recurrence_90_days_group": "0 20 2 * * * *", "active_recurring_accounts_daily_recurrence_120_days_group": "0 40 2 * * * *", diff --git a/stats/stats-server/src/runtime_setup.rs b/stats/stats-server/src/runtime_setup.rs index 976613de4..d15c670dc 100644 --- a/stats/stats-server/src/runtime_setup.rs +++ b/stats/stats-server/src/runtime_setup.rs @@ -275,6 +275,7 @@ impl RuntimeSetup { Arc::new(ActiveRecurringAccountsMonthlyRecurrence120DaysGroup), Arc::new(ActiveRecurringAccountsWeeklyRecurrence120DaysGroup), Arc::new(ActiveRecurringAccountsYearlyRecurrence120DaysGroup), + Arc::new(NewTxnsWindowGroup), // singletons but not really (include all resolutions of the same chart) Arc::new(AverageBlockRewardsGroup), Arc::new(AverageBlockSizeGroup), diff --git a/stats/stats-server/tests/it/lines.rs b/stats/stats-server/tests/it/lines.rs index b3cce84eb..90e0552c2 100644 --- a/stats/stats-server/tests/it/lines.rs +++ b/stats/stats-server/tests/it/lines.rs @@ -69,6 +69,10 @@ async fn test_lines_ok() { let mut enabled_resolutions = enabled_resolutions(line_charts).await; + // does not return data for latest dates, + // so todo: test later with other main page stuff + enabled_resolutions.remove("newTxnsWindow"); + for line_name in [ "accountsGrowth", "activeAccounts", @@ -88,6 +92,7 @@ async fn test_lines_ok() { "newBlocks", "newNativeCoinTransfers", "newTxns", + // "newTxnsWindow", "txnsFee", "txnsGrowth", // "newOperationalTxns", @@ -122,7 +127,7 @@ async fn test_lines_ok() { assert!( !chart_data.is_empty(), - "chart '{line_name}' '{resolution}' is empty" + "chart data for '{line_name}' '{resolution}' is empty" ); let info = chart diff --git a/stats/stats/src/charts/db_interaction/write.rs b/stats/stats/src/charts/db_interaction/write.rs index f5706889d..fde0e3752 100644 --- a/stats/stats/src/charts/db_interaction/write.rs +++ b/stats/stats/src/charts/db_interaction/write.rs @@ -56,6 +56,14 @@ where Ok(()) } +pub async fn clear_all_chart_data(db: &C, chart_id: i32) -> Result<(), DbErr> { + chart_data::Entity::delete_many() + .filter(chart_data::Column::ChartId.eq(chart_id)) + .exec(db) + .await?; + Ok(()) +} + pub async fn set_last_updated_at( chart_id: i32, db: &DatabaseConnection, diff --git a/stats/stats/src/charts/lines/mod.rs b/stats/stats/src/charts/lines/mod.rs index 2df999fb6..2ec4a719d 100644 --- a/stats/stats/src/charts/lines/mod.rs +++ b/stats/stats/src/charts/lines/mod.rs @@ -18,6 +18,7 @@ mod new_native_coin_holders; mod new_native_coin_transfers; mod new_operational_txns; mod new_txns; +mod new_txns_window; mod new_verified_contracts; mod operational_txns_growth; mod txns_fee; @@ -96,6 +97,7 @@ pub use new_operational_txns::{ }; pub(crate) use new_txns::NewTxnsStatement; pub use new_txns::{NewTxns, NewTxnsInt, NewTxnsMonthly, NewTxnsWeekly, NewTxnsYearly}; +pub use new_txns_window::NewTxnsWindow; pub use new_verified_contracts::{ NewVerifiedContracts, NewVerifiedContractsMonthly, NewVerifiedContractsWeekly, NewVerifiedContractsYearly, diff --git a/stats/stats/src/charts/lines/new_txns_window.rs b/stats/stats/src/charts/lines/new_txns_window.rs new file mode 100644 index 000000000..3d9b6327b --- /dev/null +++ b/stats/stats/src/charts/lines/new_txns_window.rs @@ -0,0 +1,199 @@ +//! New transactions for the last N days (usually 30). +//! +//! Allowed to work on a non-indexed networks, as it +//! recalculates whole N day window/range each time. +//! +//! Does not include last day, even as incomplete day. + +use crate::{ + data_source::{ + kinds::{ + local_db::{ + parameters::{ + update::batching::{ + parameters::{BatchMaxDays, ClearAllAndPassStep}, + BatchUpdate, + }, + DefaultCreate, DefaultQueryVec, + }, + LocalDbChartSource, + }, + remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, + }, + types::BlockscoutMigrations, + UpdateContext, + }, + range::UniversalRange, + types::{Timespan, TimespanDuration, TimespanValue}, + utils::day_start, + ChartError, ChartProperties, Named, +}; + +use chrono::{DateTime, NaiveDate, Utc}; +use entity::sea_orm_active_enums::ChartType; +use sea_orm::{FromQueryResult, Statement}; + +use super::NewTxnsStatement; + +pub const WINDOW: u64 = 30; + +fn new_txns_window_statement( + update_day: NaiveDate, + completed_migrations: &BlockscoutMigrations, +) -> Statement { + // `update_day` is not included because the data would + // be incomplete. + let window = + day_start(&update_day.saturating_sub(TimespanDuration::from_timespan_repeats(WINDOW))) + ..day_start(&update_day); + NewTxnsStatement::get_statement(Some(window), completed_migrations) +} + +pub struct NewTxnsWindowQuery; + +impl RemoteQueryBehaviour for NewTxnsWindowQuery { + type Output = Vec>; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result>, ChartError> { + let update_day = cx.time.date_naive(); + let query = new_txns_window_statement(update_day, &cx.blockscout_applied_migrations); + let mut data = TimespanValue::::find_by_statement(query) + .all(cx.blockscout.connection.as_ref()) + .await + .map_err(ChartError::BlockscoutDB)?; + // linear time for sorted sequences + data.sort_unstable_by(|a, b| a.timespan.cmp(&b.timespan)); + Ok(data) + } +} + +pub type NewTxnsWindowRemote = RemoteDatabaseSource; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "newTxnsWindow".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Line + } +} + +pub type NewTxnsWindow = LocalDbChartSource< + NewTxnsWindowRemote, + (), + DefaultCreate, + BatchUpdate< + NewTxnsWindowRemote, + (), + ClearAllAndPassStep, + BatchMaxDays, + DefaultQueryVec, + Properties, + >, + DefaultQueryVec, + Properties, +>; + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + + use super::*; + use crate::{ + data_source::{DataSource, UpdateParameters}, + query_dispatch::QuerySerialized, + tests::{ + mock_blockscout::{fill_mock_blockscout_data, imitate_reindex}, + point_construction::dt, + simple_test::{chart_output_to_expected, map_str_tuple_to_owned, prepare_chart_test}, + }, + utils::MarkedDbConnection, + }; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_txns_window_clears_and_overwrites() { + let (init_time, db, blockscout) = + prepare_chart_test::("update_txns_window_clears_and_overwrites", None) + .await; + { + let current_date = init_time.date_naive(); + fill_mock_blockscout_data(&blockscout, current_date).await; + } + let current_time = dt("2022-12-01T00:00:00").and_utc(); + + let mut parameters = UpdateParameters { + db: &MarkedDbConnection::from_test_db(&db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: Some(current_time), + force_full: false, + }; + let cx = UpdateContext::from_params_now_or_override(parameters.clone()); + NewTxnsWindow::update_recursively(&cx).await.unwrap(); + assert_eq!( + &chart_output_to_expected( + NewTxnsWindow::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), + &map_str_tuple_to_owned(vec![ + ("2022-11-09", "5"), + ("2022-11-10", "12"), + ("2022-11-11", "14"), + ("2022-11-12", "5"), + // update day is not included + ]), + ); + + let current_time = dt("2022-12-10T00:00:00").and_utc(); + parameters.update_time_override = Some(current_time); + let cx = UpdateContext::from_params_now_or_override(parameters.clone()); + NewTxnsWindow::update_recursively(&cx).await.unwrap(); + assert_eq!( + &chart_output_to_expected( + NewTxnsWindow::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), + &map_str_tuple_to_owned(vec![ + // values outside the window are removed + ("2022-11-10", "12"), + ("2022-11-11", "14"), + ("2022-11-12", "5"), + ("2022-12-01", "5"), + ]), + ); + + imitate_reindex(&blockscout, init_time.date_naive()).await; + + let current_time = dt("2022-12-11T00:00:00").and_utc(); + parameters.update_time_override = Some(current_time); + let cx = UpdateContext::from_params_now_or_override(parameters); + NewTxnsWindow::update_recursively(&cx).await.unwrap(); + assert_eq!( + &chart_output_to_expected( + NewTxnsWindow::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), + &map_str_tuple_to_owned(vec![ + // values outside the window are removed + // new values within the window are added + ("2022-11-11", "18"), + ("2022-11-12", "5"), + ("2022-12-01", "5"), + ]), + ); + } +} diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs index d4150c3c9..f35a3edc6 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs @@ -1,7 +1,7 @@ use std::future::Future; use chrono::{DateTime, Utc}; -use sea_orm::DatabaseConnection; +use sea_orm::{ConnectionTrait, TransactionTrait}; use crate::{ types::{Timespan, TimespanValue}, @@ -17,8 +17,8 @@ where /// Update chart with data from its dependencies. /// /// Returns how many records were found - fn batch_update_values_step_with( - db: &DatabaseConnection, + fn batch_update_values_step_with( + db: &C, chart_id: i32, update_time: DateTime, min_blockscout_block: i64, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs index ffaad7478..6bab0759f 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs @@ -4,7 +4,7 @@ use std::{fmt::Display, marker::PhantomData, ops::Add, str::FromStr}; use chrono::{DateTime, Utc}; use rust_decimal::prelude::Zero; -use sea_orm::DatabaseConnection; +use sea_orm::{ConnectionTrait, TransactionTrait}; use crate::{ data_source::kinds::local_db::parameters::update::batching::parameter_traits::BatchStepBehaviour, @@ -29,15 +29,18 @@ where ::Err: Display, ChartProps: ChartProperties, { - async fn batch_update_values_step_with( - db: &DatabaseConnection, + async fn batch_update_values_step_with( + db: &C, chart_id: i32, update_time: DateTime, min_blockscout_block: i64, last_accurate_point: TimespanValue, main_data: Vec>, _resolution_data: (), - ) -> Result { + ) -> Result + where + C: ConnectionTrait + TransactionTrait, + { let partial_sum = last_accurate_point.value.parse::().map_err(|e| { ChartError::Internal(format!( "failed to parse value in chart '{}': {e}", diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs index 0d0cc61ec..0714a1266 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; use chrono::{DateTime, NaiveDate, Utc}; -use sea_orm::DatabaseConnection; +use sea_orm::{ConnectionTrait, TransactionTrait}; use crate::{ data_source::kinds::local_db::parameters::update::batching::parameter_traits::BatchStepBehaviour, @@ -32,15 +32,18 @@ impl BatchStepBehaviour>, ()> where StepsRecorder: Recorder>, ()>>, { - async fn batch_update_values_step_with( - db: &DatabaseConnection, + async fn batch_update_values_step_with( + db: &C, chart_id: i32, update_time: DateTime, min_blockscout_block: i64, last_accurate_point: DateValue, main_data: Vec>, resolution_data: (), - ) -> Result { + ) -> Result + where + C: ConnectionTrait + TransactionTrait, + { StepsRecorder::record(StepInput { chart_id, update_time, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs index 41e274fd8..3d65a8545 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs @@ -1,8 +1,8 @@ use chrono::{DateTime, NaiveDate, Utc}; -use sea_orm::DatabaseConnection; +use sea_orm::{ConnectionTrait, TransactionTrait}; use crate::{ - charts::db_interaction::write::insert_data_many, + charts::db_interaction::write::{clear_all_chart_data, insert_data_many}, gettable_const, types::{ timespans::{Month, Week, Year}, @@ -34,8 +34,8 @@ impl BatchStepBehaviour( + db: &C, chart_id: i32, _update_time: DateTime, min_blockscout_block: i64, @@ -60,3 +60,43 @@ where Ok(found) } } + +/// Batch update "step" that clears all data for this chart +/// and inserts the newly queried data. +/// +/// Since it removes all data each time, it makes sense to +/// use it only with max batch size. +pub struct ClearAllAndPassStep; + +impl BatchStepBehaviour>, ()> + for ClearAllAndPassStep +where + Resolution: Timespan + Clone + Send + Sync, +{ + async fn batch_update_values_step_with( + db: &C, + chart_id: i32, + _update_time: DateTime, + min_blockscout_block: i64, + _last_accurate_point: TimespanValue, + main_data: Vec>, + _resolution_data: (), + ) -> Result { + let db = db.begin().await.map_err(ChartError::StatsDB)?; + clear_all_chart_data(&db, chart_id) + .await + .map_err(ChartError::StatsDB)?; + let result = PassVecStep::batch_update_values_step_with( + &db, + chart_id, + _update_time, + min_blockscout_block, + _last_accurate_point, + main_data, + _resolution_data, + ) + .await?; + db.commit().await.map_err(ChartError::StatsDB)?; + Ok(result) + } +} diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index f48419e20..35284bbb5 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, ops::Range, str::FromStr, sync::Arc}; use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{DatabaseConnection, DbBackend, Statement}; +use sea_orm::{ConnectionTrait, DbBackend, Statement, TransactionTrait}; use tokio::sync::Mutex; use super::{ @@ -209,15 +209,18 @@ struct ContractsGrowthCustomBatchStepBehaviour; impl BatchStepBehaviour>, ()> for ContractsGrowthCustomBatchStepBehaviour { - async fn batch_update_values_step_with( - _db: &DatabaseConnection, + async fn batch_update_values_step_with( + _db: &C, _chart_id: i32, _update_time: DateTime, _min_blockscout_block: i64, _last_accurate_point: DateValue, _main_data: Vec>, _resolution_data: (), - ) -> Result { + ) -> Result + where + C: ConnectionTrait + TransactionTrait, + { // do something (just an example, not intended for running) todo!(); // save data diff --git a/stats/stats/src/tests/mock_blockscout.rs b/stats/stats/src/tests/mock_blockscout.rs index 1803225ad..363d3052d 100644 --- a/stats/stats/src/tests/mock_blockscout.rs +++ b/stats/stats/src/tests/mock_blockscout.rs @@ -54,34 +54,13 @@ pub async fn fill_mock_blockscout_data(blockscout: &DatabaseConnection, max_date .await .unwrap(); - let blocks = vec![ - "2022-11-09T23:59:59", - "2022-11-10T00:00:00", - "2022-11-10T12:00:00", - "2022-11-10T23:59:59", - "2022-11-11T00:00:00", - "2022-11-11T12:00:00", - "2022-11-11T15:00:00", - "2022-11-11T23:59:59", - "2022-11-12T00:00:00", - "2022-12-01T10:00:00", - "2023-01-01T10:00:00", - "2023-02-01T10:00:00", - "2023-03-01T10:00:00", - ] - .into_iter() - .filter(|val| NaiveDateTime::from_str(val).unwrap().date() <= max_date) - .enumerate() - .map(|(ind, ts)| mock_block(ind as i64, NaiveDateTime::from_str(ts).unwrap(), true)) - .collect::>(); + let blocks = mock_blocks(max_date); blocks::Entity::insert_many(blocks.clone()) .exec(blockscout) .await .unwrap(); - let accounts = (1..9) - .map(|seed| mock_address(seed, false, false)) - .collect::>(); + let accounts = mock_addresses(); addresses::Entity::insert_many(accounts.clone()) .exec(blockscout) .await @@ -114,39 +93,7 @@ pub async fn fill_mock_blockscout_data(blockscout: &DatabaseConnection, max_date let failed_block = blocks.last().unwrap(); - let txns = blocks[0..blocks.len() - 1] - .iter() - // make 1/3 of blocks empty - .filter(|b| b.number.as_ref() % 3 != 1) - // add 3 transactions to block - .flat_map(|b| { - [ - mock_transaction( - b, - 21_000, - (b.number.as_ref() * 1_123_456_789) % 70_000_000_000, - &accounts, - 0, - TxType::Transfer, - ), - mock_transaction( - b, - 21_000, - (b.number.as_ref() * 1_123_456_789) % 70_000_000_000, - &accounts, - 1, - TxType::Transfer, - ), - mock_transaction( - b, - 21_000, - (b.number.as_ref() * 1_123_456_789) % 70_000_000_000, - &accounts, - 2, - TxType::ContractCall, - ), - ] - }); + let txns = mock_transactions(&blocks, &accounts); transactions::Entity::insert_many(txns) .exec(blockscout) .await @@ -309,6 +256,17 @@ pub async fn fill_mock_blockscout_data(blockscout: &DatabaseConnection, max_date .unwrap(); } +/// Expected `max_date` to be the same that was passed to `fill_mock_blockscout_data` +pub async fn imitate_reindex(blockscout: &DatabaseConnection, max_date: NaiveDate) { + let existing_blocks = mock_blocks(max_date); + let existing_accounts = mock_addresses(); + let new_txns: Vec<_> = reindexing_mock_txns(&existing_blocks, &existing_accounts); + transactions::Entity::insert_many(new_txns) + .exec(blockscout) + .await + .unwrap(); +} + /// `block_times` - block time for each block from the 2nd to the latest. /// /// ` = + 1` @@ -334,6 +292,29 @@ pub async fn fill_many_blocks( .unwrap(); } +fn mock_blocks(max_date: NaiveDate) -> Vec { + vec![ + "2022-11-09T23:59:59", + "2022-11-10T00:00:00", + "2022-11-10T12:00:00", + "2022-11-10T23:59:59", + "2022-11-11T00:00:00", + "2022-11-11T12:00:00", + "2022-11-11T15:00:00", + "2022-11-11T23:59:59", + "2022-11-12T00:00:00", + "2022-12-01T10:00:00", + "2023-01-01T10:00:00", + "2023-02-01T10:00:00", + "2023-03-01T10:00:00", + ] + .into_iter() + .filter(|val| NaiveDateTime::from_str(val).unwrap().date() <= max_date) + .enumerate() + .map(|(ind, ts)| mock_block(ind as i64, NaiveDateTime::from_str(ts).unwrap(), true)) + .collect::>() +} + fn mock_block(index: i64, ts: NaiveDateTime, consensus: bool) -> blocks::ActiveModel { let size = (1000 + (index * 15485863) % 5000) as i32; let gas_limit = if index <= 3 { 12_500_000 } else { 30_000_000 }; @@ -354,6 +335,12 @@ fn mock_block(index: i64, ts: NaiveDateTime, consensus: bool) -> blocks::ActiveM } } +fn mock_addresses() -> Vec { + (1..9) + .map(|seed| mock_address(seed, false, false)) + .collect::>() +} + fn mock_address(seed: i64, is_contract: bool, is_verified: bool) -> addresses::ActiveModel { let mut hash = seed.to_le_bytes().to_vec(); hash.extend(std::iter::repeat(0).take(32 - hash.len())); @@ -385,6 +372,80 @@ impl TxType { } } +fn mock_transactions( + blocks: &[blocks::ActiveModel], + accounts: &[addresses::ActiveModel], +) -> Vec { + blocks[0..blocks.len() - 1] + .iter() + // make 1/3 of blocks empty + .filter(|b| b.number.as_ref() % 3 != 1) + // add 3 transactions to block + .flat_map(|b| { + [ + mock_transaction( + b, + 21_000, + (b.number.as_ref() * 1_123_456_789) % 70_000_000_000, + accounts, + 0, + TxType::Transfer, + ), + mock_transaction( + b, + 21_000, + (b.number.as_ref() * 1_123_456_789) % 70_000_000_000, + accounts, + 1, + TxType::Transfer, + ), + mock_transaction( + b, + 21_000, + (b.number.as_ref() * 1_123_456_789) % 70_000_000_000, + accounts, + 2, + TxType::ContractCall, + ), + ] + }) + .collect() +} + +fn reindexing_mock_txns( + blocks: &[blocks::ActiveModel], + accounts: &[addresses::ActiveModel], +) -> Vec { + // not sure if can actually happen in blockscout, but let's + // say empty blocks got their own transactions + blocks[0..blocks.len() - 1] + .iter() + // fill in the empty blocks + .filter(|b| b.number.as_ref() % 3 == 1) + // add 2 transactions to block + .flat_map(|b| { + [ + mock_transaction( + b, + 23_000, + (b.number.as_ref() * 1_123_456_789) % 70_000_000_000, + accounts, + 0, + TxType::Transfer, + ), + mock_transaction( + b, + 23_000, + (b.number.as_ref() * 1_123_456_789) % 70_000_000_000, + accounts, + 1, + TxType::Transfer, + ), + ] + }) + .collect() +} + fn mock_transaction( block: &blocks::ActiveModel, gas: i64, @@ -581,3 +642,39 @@ fn mock_migration(name: &str, completed: Option) -> migrations_status::Act updated_at: Set(Default::default()), } } + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use itertools::Itertools; + use pretty_assertions::assert_eq; + + use super::*; + + fn assert_block_hashes_do_not_overlap( + existing_txns: &[transactions::ActiveModel], + new_txns: &[transactions::ActiveModel], + ) { + let existing_blocks: HashSet<_> = existing_txns + .iter() + .map(|t| t.block_hash.clone().unwrap().unwrap()) + .collect(); + let new_blocks: HashSet<_> = new_txns + .iter() + .map(|t| t.block_hash.clone().unwrap().unwrap()) + .collect(); + let overlapping_blocks: Vec<&Vec> = + new_blocks.intersection(&existing_blocks).collect_vec(); + assert_eq!(overlapping_blocks, Vec::<&Vec>::new()); + } + + #[test] + fn reindexing_does_not_produce_overlapping_txns() { + let existing_blocks = mock_blocks(NaiveDate::MAX); + let existing_accounts = mock_addresses(); + let existing_txns = mock_transactions(&existing_blocks, &existing_accounts); + let new_txns: Vec<_> = reindexing_mock_txns(&existing_blocks, &existing_accounts); + assert_block_hashes_do_not_overlap(&existing_txns, &new_txns); + } +} diff --git a/stats/stats/src/tests/point_construction.rs b/stats/stats/src/tests/point_construction.rs index 52503f97d..f80ed5f43 100644 --- a/stats/stats/src/tests/point_construction.rs +++ b/stats/stats/src/tests/point_construction.rs @@ -15,6 +15,10 @@ pub fn d(date: &str) -> NaiveDate { date.parse().unwrap() } +/// Example: +/// ``` +/// dt("2023-01-01T00:00:00") +/// ``` pub fn dt(time: &str) -> NaiveDateTime { time.parse().unwrap() } diff --git a/stats/stats/src/tests/simple_test.rs b/stats/stats/src/tests/simple_test.rs index 318abe22f..6edbebfee 100644 --- a/stats/stats/src/tests/simple_test.rs +++ b/stats/stats/src/tests/simple_test.rs @@ -65,7 +65,7 @@ pub async fn simple_test_chart_with_migration_variants( } } -fn chart_output_to_expected(output: Vec) -> Vec<(String, String)> { +pub fn chart_output_to_expected(output: Vec) -> Vec<(String, String)> { output.into_iter().map(|p| (p.date, p.value)).collect() } diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index f33d72ba3..dec4cbe55 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -22,8 +22,9 @@ //! [`DailyCumulativeLocalDbChartSource`](crate::data_source::kinds::local_db::DailyCumulativeLocalDbChartSource) //! ). //! 2. Construct simple (non-sync) update groups via `construct_update_group` macro (usually done in `../update_groups.rs`) -//! 3. Create mutexes (1-1 for each chart) +//! 3. Create mutexes (1-1 for each chart) (in general automatically done in `stats-server`, see instructions there) //! 4. Create synchronous versions of groups with [`SyncUpdateGroup::new`] +//! (in general automatically done in `stats-server`, see instructions there) //! use std::{ diff --git a/stats/stats/src/update_groups.rs b/stats/stats/src/update_groups.rs index 7904bf329..9bdbb805b 100644 --- a/stats/stats/src/update_groups.rs +++ b/stats/stats/src/update_groups.rs @@ -43,6 +43,8 @@ singleton_groups!( ActiveRecurringAccountsMonthlyRecurrence120Days, ActiveRecurringAccountsWeeklyRecurrence120Days, ActiveRecurringAccountsYearlyRecurrence120Days, + // Standalone chart + NewTxnsWindow, ); construct_update_group!(AverageBlockRewardsGroup {